Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 34%

137 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-14 09:22 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from types import MappingProxyType 

32from typing import TYPE_CHECKING, AbstractSet, Dict, Iterable, Mapping, Optional, Set 

33 

34from lsst.utils import doImportType 

35from lsst.utils.classes import cached_getter 

36 

37from .. import ddl 

38from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

39from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

40from ._elements import Dimension, DimensionCombination, DimensionElement 

41from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

42 

43if TYPE_CHECKING: 

44 from ...registry.interfaces import ( 

45 Database, 

46 DatabaseDimensionRecordStorage, 

47 GovernorDimensionRecordStorage, 

48 StaticTablesContext, 

49 ) 

50 from ._governor import GovernorDimension 

51 

52 

53class DatabaseTopologicalFamily(TopologicalFamily): 

54 """Database topological family implementation. 

55 

56 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

57 `DatabaseDimensionCombination` objects that have direct database 

58 representations. 

59 

60 Parameters 

61 ---------- 

62 name : `str` 

63 Name of the family. 

64 space : `TopologicalSpace` 

65 Space in which this family's regions live. 

66 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

67 The members of this family, ordered according to the priority used 

68 in `choose` (first-choice member first). 

69 """ 

70 

71 def __init__( 

72 self, 

73 name: str, 

74 space: TopologicalSpace, 

75 *, 

76 members: NamedValueAbstractSet[DimensionElement], 

77 ): 

78 super().__init__(name, space) 

79 self.members = members 

80 

81 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

82 # Docstring inherited from TopologicalFamily. 

83 for member in self.members: 

84 if member in endpoints: 

85 return member 

86 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

87 

88 @property 

89 @cached_getter 

90 def governor(self) -> GovernorDimension: 

91 """Return `GovernorDimension` common to all members of this family. 

92 

93 (`GovernorDimension`). 

94 """ 

95 governors = set(m.governor for m in self.members) 

96 if None in governors: 

97 raise RuntimeError( 

98 f"Bad {self.space.name} family definition {self.name}: at least one member " 

99 f"in {self.members} has no GovernorDimension dependency." 

100 ) 

101 try: 

102 (result,) = governors 

103 except ValueError: 

104 raise RuntimeError( 

105 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

106 f"in {self.members}." 

107 ) from None 

108 return result # type: ignore 

109 

110 members: NamedValueAbstractSet[DimensionElement] 

111 """The members of this family, ordered according to the priority used in 

112 `choose` (first-choice member first). 

113 """ 

114 

115 

116class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

117 """A construction visitor for `DatabaseTopologicalFamily`. 

118 

119 This visitor depends on (and is thus visited after) its members. 

120 

121 Parameters 

122 ---------- 

123 name : `str` 

124 Name of the family. 

125 members : `Iterable` [ `str` ] 

126 The names of the members of this family, ordered according to the 

127 priority used in `choose` (first-choice member first). 

128 """ 

129 

130 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

131 super().__init__(name) 

132 self._space = space 

133 self._members = tuple(members) 

134 

135 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

136 # Docstring inherited from DimensionConstructionVisitor. 

137 return not others.isdisjoint(self._members) 

138 

139 def visit(self, builder: DimensionConstructionBuilder) -> None: 

140 # Docstring inherited from DimensionConstructionVisitor. 

141 members = NamedValueSet(builder.elements[name] for name in self._members) 

142 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

143 builder.topology[self._space].add(family) 

144 for member in members: 

145 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination)) 

146 other = member._topology.setdefault(self._space, family) 

147 if other is not family: 

148 raise RuntimeError( 

149 f"{member.name} is declared to be a member of (at least) two " 

150 f"{self._space.name} families: {other.name} and {family.name}." 

151 ) 

152 

153 

154class DatabaseDimensionElement(DimensionElement): 

155 """An intermediate base class for `DimensionElement` database classes. 

156 

157 Theese classes are ones whose instances map directly to a database 

158 table or query. 

159 

160 Parameters 

161 ---------- 

162 name : `str` 

163 Name of the dimension. 

164 storage : `dict` 

165 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

166 that will back this element in the registry (in a "cls" key) along 

167 with any other construction keyword arguments (in other keys). 

168 implied : `NamedValueAbstractSet` [ `Dimension` ] 

169 Other dimensions whose keys are included in this dimension's (logical) 

170 table as foreign keys. 

171 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

172 Field specifications for all non-key fields in this dimension's table. 

173 """ 

174 

175 def __init__( 

176 self, 

177 name: str, 

178 storage: dict, 

179 *, 

180 implied: NamedValueAbstractSet[Dimension], 

181 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

182 ): 

183 self._name = name 

184 self._storage = storage 

185 self._implied = implied 

186 self._metadata = metadata 

187 self._topology: Dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

188 

189 @property 

190 def name(self) -> str: 

191 # Docstring inherited from TopologicalRelationshipEndpoint. 

192 return self._name 

193 

194 @property 

195 def implied(self) -> NamedValueAbstractSet[Dimension]: 

196 # Docstring inherited from DimensionElement. 

197 return self._implied 

198 

199 @property 

200 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

201 # Docstring inherited from DimensionElement. 

202 return self._metadata 

203 

204 @property 

205 def viewOf(self) -> Optional[str]: 

206 # Docstring inherited from DimensionElement. 

207 # This is a bit encapsulation-breaking; these storage config values 

208 # are supposed to be opaque here, and just forwarded on to some 

209 # DimensionRecordStorage implementation. The long-term fix is to 

210 # move viewOf entirely, by changing the code that relies on it to rely 

211 # on the DimensionRecordStorage object instead. 

212 storage = self._storage.get("nested") 

213 if storage is None: 

214 storage = self._storage 

215 return storage.get("view_of") 

216 

217 @property 

218 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

219 # Docstring inherited from TopologicalRelationshipEndpoint 

220 return MappingProxyType(self._topology) 

221 

222 @property 

223 def spatial(self) -> Optional[DatabaseTopologicalFamily]: 

224 # Docstring inherited from TopologicalRelationshipEndpoint 

225 return self.topology.get(TopologicalSpace.SPATIAL) 

226 

227 @property 

228 def temporal(self) -> Optional[DatabaseTopologicalFamily]: 

229 # Docstring inherited from TopologicalRelationshipEndpoint 

230 return self.topology.get(TopologicalSpace.TEMPORAL) 

231 

232 def makeStorage( 

233 self, 

234 db: Database, 

235 *, 

236 context: Optional[StaticTablesContext] = None, 

237 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

238 view_target: DatabaseDimensionRecordStorage | None = None, 

239 ) -> DatabaseDimensionRecordStorage: 

240 """Make the dimension record storage instance for this database. 

241 

242 Constructs the `DimensionRecordStorage` instance that should 

243 be used to back this element in a registry. 

244 

245 Parameters 

246 ---------- 

247 db : `Database` 

248 Interface to the underlying database engine and namespace. 

249 context : `StaticTablesContext`, optional 

250 If provided, an object to use to create any new tables. If not 

251 provided, ``db.ensureTableExists`` should be used instead. 

252 governors : `NamedKeyMapping` 

253 Mapping from `GovernorDimension` to the record storage backend for 

254 that dimension, containing all governor dimensions. 

255 view_target : `DatabaseDimensionRecordStorage`, optional 

256 Storage object for the element this target's storage is a view of 

257 (i.e. when `viewOf` is not `None`). 

258 

259 Returns 

260 ------- 

261 storage : `DatabaseDimensionRecordStorage` 

262 Storage object that should back this element in a registry. 

263 """ 

264 from ...registry.interfaces import DatabaseDimensionRecordStorage 

265 

266 cls = doImportType(self._storage["cls"]) 

267 assert issubclass(cls, DatabaseDimensionRecordStorage) 

268 return cls.initialize( 

269 db, 

270 self, 

271 context=context, 

272 config=self._storage, 

273 governors=governors, 

274 view_target=view_target, 

275 ) 

276 

277 

278class DatabaseDimension(Dimension, DatabaseDimensionElement): 

279 """A `Dimension` class that maps directly to a database table or query. 

280 

281 Parameters 

282 ---------- 

283 name : `str` 

284 Name of the dimension. 

285 storage : `dict` 

286 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

287 that will back this element in the registry (in a "cls" key) along 

288 with any other construction keyword arguments (in other keys). 

289 required : `NamedValueSet` [ `Dimension` ] 

290 Other dimensions whose keys are part of the compound primary key for 

291 this dimension's (logical) table, as well as references to their own 

292 tables. The ``required`` parameter does not include ``self`` (it 

293 can't, of course), but the corresponding attribute does - it is added 

294 by the constructor. 

295 implied : `NamedValueAbstractSet` [ `Dimension` ] 

296 Other dimensions whose keys are included in this dimension's (logical) 

297 table as foreign keys. 

298 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

299 Field specifications for all non-key fields in this dimension's table. 

300 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

301 Fields that can each be used to uniquely identify this dimension (given 

302 values for all required dimensions). The first of these is used as 

303 (part of) this dimension's table's primary key, while others are used 

304 to define unique constraints. 

305 

306 Notes 

307 ----- 

308 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

309 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

310 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

311 members. 

312 """ 

313 

314 def __init__( 

315 self, 

316 name: str, 

317 storage: dict, 

318 *, 

319 required: NamedValueSet[Dimension], 

320 implied: NamedValueAbstractSet[Dimension], 

321 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

322 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

323 ): 

324 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

325 required.add(self) 

326 self._required = required.freeze() 

327 self._uniqueKeys = uniqueKeys 

328 

329 @property 

330 def required(self) -> NamedValueAbstractSet[Dimension]: 

331 # Docstring inherited from DimensionElement. 

332 return self._required 

333 

334 @property 

335 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

336 # Docstring inherited from Dimension. 

337 return self._uniqueKeys 

338 

339 

340class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

341 """A combination class that maps directly to a database table or query. 

342 

343 Parameters 

344 ---------- 

345 name : `str` 

346 Name of the dimension. 

347 storage : `dict` 

348 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

349 that will back this element in the registry (in a "cls" key) along 

350 with any other construction keyword arguments (in other keys). 

351 required : `NamedValueAbstractSet` [ `Dimension` ] 

352 Dimensions whose keys define the compound primary key for this 

353 combinations's (logical) table, as well as references to their own 

354 tables. 

355 implied : `NamedValueAbstractSet` [ `Dimension` ] 

356 Dimensions whose keys are included in this combinations's (logical) 

357 table as foreign keys. 

358 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

359 Field specifications for all non-key fields in this combination's 

360 table. 

361 alwaysJoin : `bool`, optional 

362 If `True`, always include this element in any query or data ID in 

363 which its ``required`` dimensions appear, because it defines a 

364 relationship between those dimensions that must always be satisfied. 

365 

366 Notes 

367 ----- 

368 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

369 but it is the responsibility of 

370 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

371 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

372 

373 This class has a lot in common with `DatabaseDimension`, but they are 

374 expected to diverge in future changes, and the only way to make them share 

375 method implementations would be via multiple inheritance. Given the 

376 trivial nature of all of those implementations, this does not seem worth 

377 the drawbacks (particularly the constraints it imposes on constructor 

378 signatures). 

379 """ 

380 

381 def __init__( 

382 self, 

383 name: str, 

384 storage: dict, 

385 *, 

386 required: NamedValueAbstractSet[Dimension], 

387 implied: NamedValueAbstractSet[Dimension], 

388 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

389 alwaysJoin: bool, 

390 ): 

391 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

392 self._required = required 

393 self._alwaysJoin = alwaysJoin 

394 

395 @property 

396 def required(self) -> NamedValueAbstractSet[Dimension]: 

397 # Docstring inherited from DimensionElement. 

398 return self._required 

399 

400 @property 

401 def alwaysJoin(self) -> bool: 

402 # Docstring inherited from DimensionElement. 

403 return self._alwaysJoin 

404 

405 

406class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

407 """Construction visitor for database dimension and dimension combination. 

408 

409 Specifically, a construction visitor for `DatabaseDimension` and 

410 `DatabaseDimensionCombination`. 

411 

412 Parameters 

413 ---------- 

414 name : `str` 

415 Name of the dimension. 

416 storage : `dict` 

417 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

418 that will back this element in the registry (in a "cls" key) along 

419 with any other construction keyword arguments (in other keys). 

420 required : `Set` [ `Dimension` ] 

421 Names of dimensions whose keys define the compound primary key for this 

422 element's (logical) table, as well as references to their own 

423 tables. 

424 implied : `Set` [ `Dimension` ] 

425 Names of dimension whose keys are included in this elements's 

426 (logical) table as foreign keys. 

427 metadata : `Iterable` [ `ddl.FieldSpec` ] 

428 Field specifications for all non-key fields in this element's table. 

429 uniqueKeys : `Iterable` [ `ddl.FieldSpec` ] 

430 Fields that can each be used to uniquely identify this dimension (given 

431 values for all required dimensions). The first of these is used as 

432 (part of) this dimension's table's primary key, while others are used 

433 to define unique constraints. Should be empty for 

434 `DatabaseDimensionCombination` definitions. 

435 alwaysJoin : `bool`, optional 

436 If `True`, always include this element in any query or data ID in 

437 which its ``required`` dimensions appear, because it defines a 

438 relationship between those dimensions that must always be satisfied. 

439 Should only be provided when a `DimensionCombination` is being 

440 constructed. 

441 """ 

442 

443 def __init__( 

444 self, 

445 name: str, 

446 storage: dict, 

447 required: Set[str], 

448 implied: Set[str], 

449 metadata: Iterable[ddl.FieldSpec] = (), 

450 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

451 alwaysJoin: bool = False, 

452 ): 

453 super().__init__(name) 

454 self._storage = storage 

455 self._required = required 

456 self._implied = implied 

457 self._metadata = NamedValueSet(metadata).freeze() 

458 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

459 self._alwaysJoin = alwaysJoin 

460 

461 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

462 # Docstring inherited from DimensionConstructionVisitor. 

463 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

464 

465 def visit(self, builder: DimensionConstructionBuilder) -> None: 

466 # Docstring inherited from DimensionConstructionVisitor. 

467 # Expand required dependencies. 

468 for name in tuple(self._required): # iterate over copy 

469 self._required.update(builder.dimensions[name].required.names) 

470 # Transform required and implied Dimension names into instances, 

471 # and reorder to match builder's order. 

472 required: NamedValueSet[Dimension] = NamedValueSet() 

473 implied: NamedValueSet[Dimension] = NamedValueSet() 

474 for dimension in builder.dimensions: 

475 if dimension.name in self._required: 

476 required.add(dimension) 

477 if dimension.name in self._implied: 

478 implied.add(dimension) 

479 

480 if self._uniqueKeys: 

481 if self._alwaysJoin: 

482 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

483 # Special handling for creating Dimension instances. 

484 dimension = DatabaseDimension( 

485 self.name, 

486 storage=self._storage, 

487 required=required, 

488 implied=implied.freeze(), 

489 metadata=self._metadata, 

490 uniqueKeys=self._uniqueKeys, 

491 ) 

492 builder.dimensions.add(dimension) 

493 builder.elements.add(dimension) 

494 else: 

495 # Special handling for creating DimensionCombination instances. 

496 combination = DatabaseDimensionCombination( 

497 self.name, 

498 storage=self._storage, 

499 required=required, 

500 implied=implied.freeze(), 

501 metadata=self._metadata, 

502 alwaysJoin=self._alwaysJoin, 

503 ) 

504 builder.elements.add(combination)