Coverage for python/lsst/daf/butler/registry/interfaces/_obscore.py: 98%

28 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 15:14 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Interfaces for classes that manage obscore table(s) in a `Registry`. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = ["ObsCoreTableManager"] 

28 

29from abc import abstractmethod 

30from collections.abc import Iterable, Iterator, Mapping 

31from contextlib import contextmanager 

32from typing import TYPE_CHECKING, Any 

33 

34import sqlalchemy 

35 

36from ._versioning import VersionedExtension, VersionTuple 

37 

38if TYPE_CHECKING: 

39 from lsst.sphgeom import Region 

40 

41 from ...core import DatasetRef, DimensionUniverse 

42 from ..queries import SqlQueryContext 

43 from ._collections import CollectionRecord 

44 from ._database import Database, StaticTablesContext 

45 from ._datasets import DatasetRecordStorageManager 

46 from ._dimensions import DimensionRecordStorageManager 

47 

48 

49class ObsCoreTableManager(VersionedExtension): 

50 """An interface for populating ObsCore tables(s).""" 

51 

52 def __init__(self, *, registry_schema_version: VersionTuple | None = None): 

53 super().__init__(registry_schema_version=registry_schema_version) 

54 

55 @classmethod 

56 @abstractmethod 

57 def initialize( 

58 cls, 

59 db: Database, 

60 context: StaticTablesContext, 

61 *, 

62 universe: DimensionUniverse, 

63 config: Mapping, 

64 datasets: type[DatasetRecordStorageManager], 

65 dimensions: DimensionRecordStorageManager, 

66 registry_schema_version: VersionTuple | None = None, 

67 ) -> ObsCoreTableManager: 

68 """Construct an instance of the manager. 

69 

70 Parameters 

71 ---------- 

72 db : `Database` 

73 Interface to the underlying database engine and namespace. 

74 context : `StaticTablesContext` 

75 Context object obtained from `Database.declareStaticTables`; used 

76 to declare any tables that should always be present in a layer 

77 implemented with this manager. 

78 universe : `DimensionUniverse` 

79 All dimensions known to the registry. 

80 config : `dict` [ `str`, `Any` ] 

81 Configuration of the obscore manager. 

82 datasets : `type` 

83 Type of dataset manager. 

84 dimensions: `DimensionRecordStorageManager` 

85 Manager for Registry dimensions. 

86 registry_schema_version : `VersionTuple` or `None` 

87 Schema version of this extension as defined in registry. 

88 

89 Returns 

90 ------- 

91 manager : `ObsCoreTableManager` 

92 An instance of a concrete `ObsCoreTableManager` subclass. 

93 """ 

94 raise NotImplementedError() 

95 

96 @abstractmethod 

97 def config_json(self) -> str: 

98 """Dump configuration in JSON format. 

99 

100 Returns 

101 ------- 

102 json : `str` 

103 Configuration serialized in JSON format. 

104 """ 

105 raise NotImplementedError() 

106 

107 @abstractmethod 

108 def add_datasets(self, refs: Iterable[DatasetRef], context: SqlQueryContext) -> int: 

109 """Possibly add datasets to the obscore table. 

110 

111 This method should be called when new datasets are added to a RUN 

112 collection. 

113 

114 Parameters 

115 ---------- 

116 refs : `iterable` [ `DatasetRef` ] 

117 Dataset refs to add. Dataset refs have to be completely expanded. 

118 If a record with the same dataset ID is already in obscore table, 

119 the dataset is ignored. 

120 context : `SqlQueryContext` 

121 Context used to execute queries for additional dimension metadata. 

122 

123 Returns 

124 ------- 

125 count : `int` 

126 Actual number of records inserted into obscore table. 

127 

128 Notes 

129 ----- 

130 Dataset data types and collection names are checked against configured 

131 list of collections and dataset types, non-matching datasets are 

132 ignored and not added to the obscore table. 

133 

134 When configuration parameter ``collection_type`` is not "RUN", this 

135 method should return immediately. 

136 

137 Note that there is no matching method to remove datasets from obscore 

138 table, we assume that removal happens via foreign key constraint to 

139 dataset table with "ON DELETE CASCADE" option. 

140 """ 

141 raise NotImplementedError() 

142 

143 @abstractmethod 

144 def associate( 

145 self, refs: Iterable[DatasetRef], collection: CollectionRecord, context: SqlQueryContext 

146 ) -> int: 

147 """Possibly add datasets to the obscore table. 

148 

149 This method should be called when existing datasets are associated with 

150 a TAGGED collection. 

151 

152 Parameters 

153 ---------- 

154 refs : `iterable` [ `DatasetRef` ] 

155 Dataset refs to add. Dataset refs have to be completely expanded. 

156 If a record with the same dataset ID is already in obscore table, 

157 the dataset is ignored. 

158 collection : `CollectionRecord` 

159 Collection record for a TAGGED collection. 

160 context : `SqlQueryContext` 

161 Context used to execute queries for additional dimension metadata. 

162 

163 Returns 

164 ------- 

165 count : `int` 

166 Actual number of records inserted into obscore table. 

167 

168 Notes 

169 ----- 

170 Dataset data types and collection names are checked against configured 

171 list of collections and dataset types, non-matching datasets are 

172 ignored and not added to the obscore table. 

173 

174 When configuration parameter ``collection_type`` is not "TAGGED", this 

175 method should return immediately. 

176 """ 

177 raise NotImplementedError() 

178 

179 @abstractmethod 

180 def disassociate(self, refs: Iterable[DatasetRef], collection: CollectionRecord) -> int: 

181 """Possibly remove datasets from the obscore table. 

182 

183 This method should be called when datasets are disassociated from a 

184 TAGGED collection. 

185 

186 Parameters 

187 ---------- 

188 refs : `iterable` [ `DatasetRef` ] 

189 Dataset refs to remove. Dataset refs have to be resolved. 

190 collection : `CollectionRecord` 

191 Collection record for a TAGGED collection. 

192 

193 Returns 

194 ------- 

195 count : `int` 

196 Actual number of records removed from obscore table. 

197 

198 Notes 

199 ----- 

200 Dataset data types and collection names are checked against configured 

201 list of collections and dataset types, non-matching datasets are 

202 ignored and not added to the obscore table. 

203 

204 When configuration parameter ``collection_type`` is not "TAGGED", this 

205 method should return immediately. 

206 """ 

207 raise NotImplementedError() 

208 

209 @abstractmethod 

210 def update_exposure_regions(self, instrument: str, region_data: Iterable[tuple[int, int, Region]]) -> int: 

211 """Update existing exposure records with spatial region data. 

212 

213 Parameters 

214 ---------- 

215 instrument : `str` 

216 Instrument name. 

217 region_data : `~collections.abc.Iterable` [`tuple` [`int`, `int`, \ 

218 `~lsst.sphgeom.Region` ]] 

219 Sequence of tuples, each tuple contains three values - exposure ID, 

220 detector ID, and corresponding region. 

221 

222 Returns 

223 ------- 

224 count : `int` 

225 Actual number of records updated. 

226 

227 Notes 

228 ----- 

229 This method is needed to update obscore records for raw exposures which 

230 are ingested before their corresponding visits are defined. Exposure 

231 records added when visit is already defined will get their regions 

232 from their matching visits automatically. 

233 """ 

234 raise NotImplementedError() 

235 

236 @abstractmethod 

237 @contextmanager 

238 def query( 

239 self, columns: Iterable[str | sqlalchemy.sql.expression.ColumnElement] | None = None, /, **kwargs: Any 

240 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

241 """Run a SELECT query against obscore table and return result rows. 

242 

243 Parameters 

244 ---------- 

245 columns : `~collections.abc.Iterable` [`str`] 

246 Columns to return from query. It is a sequence which can include 

247 column names or any other column elements (e.g. 

248 `sqlalchemy.sql.functions.count` function). 

249 **kwargs 

250 Restriction on values of individual obscore columns. Key is the 

251 column name, value is the required value of the column. Multiple 

252 restrictions are ANDed together. 

253 

254 Returns 

255 ------- 

256 result_context : `sqlalchemy.engine.CursorResult` 

257 Context manager that returns the query result object when entered. 

258 These results are invalidated when the context is exited. 

259 """ 

260 raise NotImplementedError()