Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 44%

138 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 09:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from collections.abc import Iterable, Mapping, Set 

32from types import MappingProxyType 

33from typing import TYPE_CHECKING 

34 

35from lsst.utils import doImportType 

36from lsst.utils.classes import cached_getter 

37 

38from .. import ddl 

39from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

40from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

41from ._elements import Dimension, DimensionCombination, DimensionElement 

42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

43 

44if TYPE_CHECKING: 

45 from ...registry.interfaces import ( 

46 Database, 

47 DatabaseDimensionRecordStorage, 

48 GovernorDimensionRecordStorage, 

49 StaticTablesContext, 

50 ) 

51 from ._governor import GovernorDimension 

52 

53 

54class DatabaseTopologicalFamily(TopologicalFamily): 

55 """Database topological family implementation. 

56 

57 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

58 `DatabaseDimensionCombination` objects that have direct database 

59 representations. 

60 

61 Parameters 

62 ---------- 

63 name : `str` 

64 Name of the family. 

65 space : `TopologicalSpace` 

66 Space in which this family's regions live. 

67 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

68 The members of this family, ordered according to the priority used 

69 in `choose` (first-choice member first). 

70 """ 

71 

72 def __init__( 

73 self, 

74 name: str, 

75 space: TopologicalSpace, 

76 *, 

77 members: NamedValueAbstractSet[DimensionElement], 

78 ): 

79 super().__init__(name, space) 

80 self.members = members 

81 

82 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

83 # Docstring inherited from TopologicalFamily. 

84 for member in self.members: 

85 if member in endpoints: 

86 return member 

87 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

88 

89 @property 

90 @cached_getter 

91 def governor(self) -> GovernorDimension: 

92 """Return `GovernorDimension` common to all members of this family. 

93 

94 (`GovernorDimension`). 

95 """ 

96 governors = {m.governor for m in self.members} 

97 if None in governors: 

98 raise RuntimeError( 

99 f"Bad {self.space.name} family definition {self.name}: at least one member " 

100 f"in {self.members} has no GovernorDimension dependency." 

101 ) 

102 try: 

103 (result,) = governors 

104 except ValueError: 

105 raise RuntimeError( 

106 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

107 f"in {self.members}." 

108 ) from None 

109 return result # type: ignore 

110 

111 members: NamedValueAbstractSet[DimensionElement] 

112 """The members of this family, ordered according to the priority used in 

113 `choose` (first-choice member first). 

114 """ 

115 

116 

117class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

118 """A construction visitor for `DatabaseTopologicalFamily`. 

119 

120 This visitor depends on (and is thus visited after) its members. 

121 

122 Parameters 

123 ---------- 

124 name : `str` 

125 Name of the family. 

126 members : `~collections.abc.Iterable` [ `str` ] 

127 The names of the members of this family, ordered according to the 

128 priority used in `choose` (first-choice member first). 

129 """ 

130 

131 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

132 super().__init__(name) 

133 self._space = space 

134 self._members = tuple(members) 

135 

136 def hasDependenciesIn(self, others: Set[str]) -> bool: 

137 # Docstring inherited from DimensionConstructionVisitor. 

138 return not others.isdisjoint(self._members) 

139 

140 def visit(self, builder: DimensionConstructionBuilder) -> None: 

141 # Docstring inherited from DimensionConstructionVisitor. 

142 members = NamedValueSet(builder.elements[name] for name in self._members) 

143 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

144 builder.topology[self._space].add(family) 

145 for member in members: 

146 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination)) 

147 other = member._topology.setdefault(self._space, family) 

148 if other is not family: 

149 raise RuntimeError( 

150 f"{member.name} is declared to be a member of (at least) two " 

151 f"{self._space.name} families: {other.name} and {family.name}." 

152 ) 

153 

154 

155class DatabaseDimensionElement(DimensionElement): 

156 """An intermediate base class for `DimensionElement` database classes. 

157 

158 Theese classes are ones whose instances map directly to a database 

159 table or query. 

160 

161 Parameters 

162 ---------- 

163 name : `str` 

164 Name of the dimension. 

165 storage : `dict` 

166 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

167 that will back this element in the registry (in a "cls" key) along 

168 with any other construction keyword arguments (in other keys). 

169 implied : `NamedValueAbstractSet` [ `Dimension` ] 

170 Other dimensions whose keys are included in this dimension's (logical) 

171 table as foreign keys. 

172 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

173 Field specifications for all non-key fields in this dimension's table. 

174 """ 

175 

176 def __init__( 

177 self, 

178 name: str, 

179 storage: dict, 

180 *, 

181 implied: NamedValueAbstractSet[Dimension], 

182 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

183 ): 

184 self._name = name 

185 self._storage = storage 

186 self._implied = implied 

187 self._metadata = metadata 

188 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

189 

190 @property 

191 def name(self) -> str: 

192 # Docstring inherited from TopologicalRelationshipEndpoint. 

193 return self._name 

194 

195 @property 

196 def implied(self) -> NamedValueAbstractSet[Dimension]: 

197 # Docstring inherited from DimensionElement. 

198 return self._implied 

199 

200 @property 

201 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

202 # Docstring inherited from DimensionElement. 

203 return self._metadata 

204 

205 @property 

206 def viewOf(self) -> str | None: 

207 # Docstring inherited from DimensionElement. 

208 # This is a bit encapsulation-breaking; these storage config values 

209 # are supposed to be opaque here, and just forwarded on to some 

210 # DimensionRecordStorage implementation. The long-term fix is to 

211 # move viewOf entirely, by changing the code that relies on it to rely 

212 # on the DimensionRecordStorage object instead. 

213 storage = self._storage.get("nested") 

214 if storage is None: 

215 storage = self._storage 

216 return storage.get("view_of") 

217 

218 @property 

219 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

220 # Docstring inherited from TopologicalRelationshipEndpoint 

221 return MappingProxyType(self._topology) 

222 

223 @property 

224 def spatial(self) -> DatabaseTopologicalFamily | None: 

225 # Docstring inherited from TopologicalRelationshipEndpoint 

226 return self.topology.get(TopologicalSpace.SPATIAL) 

227 

228 @property 

229 def temporal(self) -> DatabaseTopologicalFamily | None: 

230 # Docstring inherited from TopologicalRelationshipEndpoint 

231 return self.topology.get(TopologicalSpace.TEMPORAL) 

232 

233 def makeStorage( 

234 self, 

235 db: Database, 

236 *, 

237 context: StaticTablesContext | None = None, 

238 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

239 view_target: DatabaseDimensionRecordStorage | None = None, 

240 ) -> DatabaseDimensionRecordStorage: 

241 """Make the dimension record storage instance for this database. 

242 

243 Constructs the `DimensionRecordStorage` instance that should 

244 be used to back this element in a registry. 

245 

246 Parameters 

247 ---------- 

248 db : `Database` 

249 Interface to the underlying database engine and namespace. 

250 context : `StaticTablesContext`, optional 

251 If provided, an object to use to create any new tables. If not 

252 provided, ``db.ensureTableExists`` should be used instead. 

253 governors : `NamedKeyMapping` 

254 Mapping from `GovernorDimension` to the record storage backend for 

255 that dimension, containing all governor dimensions. 

256 view_target : `DatabaseDimensionRecordStorage`, optional 

257 Storage object for the element this target's storage is a view of 

258 (i.e. when `viewOf` is not `None`). 

259 

260 Returns 

261 ------- 

262 storage : `DatabaseDimensionRecordStorage` 

263 Storage object that should back this element in a registry. 

264 """ 

265 from ...registry.interfaces import DatabaseDimensionRecordStorage 

266 

267 cls = doImportType(self._storage["cls"]) 

268 assert issubclass(cls, DatabaseDimensionRecordStorage) 

269 return cls.initialize( 

270 db, 

271 self, 

272 context=context, 

273 config=self._storage, 

274 governors=governors, 

275 view_target=view_target, 

276 ) 

277 

278 

279class DatabaseDimension(Dimension, DatabaseDimensionElement): 

280 """A `Dimension` class that maps directly to a database table or query. 

281 

282 Parameters 

283 ---------- 

284 name : `str` 

285 Name of the dimension. 

286 storage : `dict` 

287 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

288 that will back this element in the registry (in a "cls" key) along 

289 with any other construction keyword arguments (in other keys). 

290 required : `NamedValueSet` [ `Dimension` ] 

291 Other dimensions whose keys are part of the compound primary key for 

292 this dimension's (logical) table, as well as references to their own 

293 tables. The ``required`` parameter does not include ``self`` (it 

294 can't, of course), but the corresponding attribute does - it is added 

295 by the constructor. 

296 implied : `NamedValueAbstractSet` [ `Dimension` ] 

297 Other dimensions whose keys are included in this dimension's (logical) 

298 table as foreign keys. 

299 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

300 Field specifications for all non-key fields in this dimension's table. 

301 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

302 Fields that can each be used to uniquely identify this dimension (given 

303 values for all required dimensions). The first of these is used as 

304 (part of) this dimension's table's primary key, while others are used 

305 to define unique constraints. 

306 

307 Notes 

308 ----- 

309 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

310 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

311 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

312 members. 

313 """ 

314 

315 def __init__( 

316 self, 

317 name: str, 

318 storage: dict, 

319 *, 

320 required: NamedValueSet[Dimension], 

321 implied: NamedValueAbstractSet[Dimension], 

322 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

323 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

324 ): 

325 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

326 required.add(self) 

327 self._required = required.freeze() 

328 self._uniqueKeys = uniqueKeys 

329 

330 @property 

331 def required(self) -> NamedValueAbstractSet[Dimension]: 

332 # Docstring inherited from DimensionElement. 

333 return self._required 

334 

335 @property 

336 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

337 # Docstring inherited from Dimension. 

338 return self._uniqueKeys 

339 

340 

341class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

342 """A combination class that maps directly to a database table or query. 

343 

344 Parameters 

345 ---------- 

346 name : `str` 

347 Name of the dimension. 

348 storage : `dict` 

349 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

350 that will back this element in the registry (in a "cls" key) along 

351 with any other construction keyword arguments (in other keys). 

352 required : `NamedValueAbstractSet` [ `Dimension` ] 

353 Dimensions whose keys define the compound primary key for this 

354 combinations's (logical) table, as well as references to their own 

355 tables. 

356 implied : `NamedValueAbstractSet` [ `Dimension` ] 

357 Dimensions whose keys are included in this combinations's (logical) 

358 table as foreign keys. 

359 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

360 Field specifications for all non-key fields in this combination's 

361 table. 

362 alwaysJoin : `bool`, optional 

363 If `True`, always include this element in any query or data ID in 

364 which its ``required`` dimensions appear, because it defines a 

365 relationship between those dimensions that must always be satisfied. 

366 

367 Notes 

368 ----- 

369 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

370 but it is the responsibility of 

371 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

372 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

373 

374 This class has a lot in common with `DatabaseDimension`, but they are 

375 expected to diverge in future changes, and the only way to make them share 

376 method implementations would be via multiple inheritance. Given the 

377 trivial nature of all of those implementations, this does not seem worth 

378 the drawbacks (particularly the constraints it imposes on constructor 

379 signatures). 

380 """ 

381 

382 def __init__( 

383 self, 

384 name: str, 

385 storage: dict, 

386 *, 

387 required: NamedValueAbstractSet[Dimension], 

388 implied: NamedValueAbstractSet[Dimension], 

389 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

390 alwaysJoin: bool, 

391 ): 

392 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

393 self._required = required 

394 self._alwaysJoin = alwaysJoin 

395 

396 @property 

397 def required(self) -> NamedValueAbstractSet[Dimension]: 

398 # Docstring inherited from DimensionElement. 

399 return self._required 

400 

401 @property 

402 def alwaysJoin(self) -> bool: 

403 # Docstring inherited from DimensionElement. 

404 return self._alwaysJoin 

405 

406 

407class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

408 """Construction visitor for database dimension and dimension combination. 

409 

410 Specifically, a construction visitor for `DatabaseDimension` and 

411 `DatabaseDimensionCombination`. 

412 

413 Parameters 

414 ---------- 

415 name : `str` 

416 Name of the dimension. 

417 storage : `dict` 

418 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

419 that will back this element in the registry (in a "cls" key) along 

420 with any other construction keyword arguments (in other keys). 

421 required : `~collections.abc.Set` [ `Dimension` ] 

422 Names of dimensions whose keys define the compound primary key for this 

423 element's (logical) table, as well as references to their own 

424 tables. 

425 implied : `~collections.abc.Set` [ `Dimension` ] 

426 Names of dimension whose keys are included in this elements's 

427 (logical) table as foreign keys. 

428 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

429 Field specifications for all non-key fields in this element's table. 

430 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

431 Fields that can each be used to uniquely identify this dimension (given 

432 values for all required dimensions). The first of these is used as 

433 (part of) this dimension's table's primary key, while others are used 

434 to define unique constraints. Should be empty for 

435 `DatabaseDimensionCombination` definitions. 

436 alwaysJoin : `bool`, optional 

437 If `True`, always include this element in any query or data ID in 

438 which its ``required`` dimensions appear, because it defines a 

439 relationship between those dimensions that must always be satisfied. 

440 Should only be provided when a `DimensionCombination` is being 

441 constructed. 

442 """ 

443 

444 def __init__( 

445 self, 

446 name: str, 

447 storage: dict, 

448 required: set[str], 

449 implied: set[str], 

450 metadata: Iterable[ddl.FieldSpec] = (), 

451 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

452 alwaysJoin: bool = False, 

453 ): 

454 super().__init__(name) 

455 self._storage = storage 

456 self._required = required 

457 self._implied = implied 

458 self._metadata = NamedValueSet(metadata).freeze() 

459 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

460 self._alwaysJoin = alwaysJoin 

461 

462 def hasDependenciesIn(self, others: Set[str]) -> bool: 

463 # Docstring inherited from DimensionConstructionVisitor. 

464 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

465 

466 def visit(self, builder: DimensionConstructionBuilder) -> None: 

467 # Docstring inherited from DimensionConstructionVisitor. 

468 # Expand required dependencies. 

469 for name in tuple(self._required): # iterate over copy 

470 self._required.update(builder.dimensions[name].required.names) 

471 # Transform required and implied Dimension names into instances, 

472 # and reorder to match builder's order. 

473 required: NamedValueSet[Dimension] = NamedValueSet() 

474 implied: NamedValueSet[Dimension] = NamedValueSet() 

475 for dimension in builder.dimensions: 

476 if dimension.name in self._required: 

477 required.add(dimension) 

478 if dimension.name in self._implied: 

479 implied.add(dimension) 

480 

481 if self._uniqueKeys: 

482 if self._alwaysJoin: 

483 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

484 # Special handling for creating Dimension instances. 

485 dimension = DatabaseDimension( 

486 self.name, 

487 storage=self._storage, 

488 required=required, 

489 implied=implied.freeze(), 

490 metadata=self._metadata, 

491 uniqueKeys=self._uniqueKeys, 

492 ) 

493 builder.dimensions.add(dimension) 

494 builder.elements.add(dimension) 

495 else: 

496 # Special handling for creating DimensionCombination instances. 

497 combination = DatabaseDimensionCombination( 

498 self.name, 

499 storage=self._storage, 

500 required=required, 

501 implied=implied.freeze(), 

502 metadata=self._metadata, 

503 alwaysJoin=self._alwaysJoin, 

504 ) 

505 builder.elements.add(combination)