Coverage for python/lsst/daf/butler/dimensions/_database.py: 44%

143 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "DatabaseDimension", 

32 "DatabaseDimensionCombination", 

33 "DatabaseDimensionElement", 

34 "DatabaseTopologicalFamily", 

35) 

36 

37from collections.abc import Iterable, Mapping, Set 

38from types import MappingProxyType 

39from typing import TYPE_CHECKING 

40 

41from lsst.utils import doImportType 

42from lsst.utils.classes import cached_getter 

43 

44from .. import ddl 

45from .._named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

46from .._topology import TopologicalFamily, TopologicalSpace 

47from ._elements import Dimension, DimensionCombination, DimensionElement 

48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

49 

50if TYPE_CHECKING: 

51 from ..registry.interfaces import ( 

52 Database, 

53 DatabaseDimensionRecordStorage, 

54 GovernorDimensionRecordStorage, 

55 StaticTablesContext, 

56 ) 

57 from ._governor import GovernorDimension 

58 from ._universe import DimensionUniverse 

59 

60 

61class DatabaseTopologicalFamily(TopologicalFamily): 

62 """Database topological family implementation. 

63 

64 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

65 `DatabaseDimensionCombination` objects that have direct database 

66 representations. 

67 

68 Parameters 

69 ---------- 

70 name : `str` 

71 Name of the family. 

72 space : `TopologicalSpace` 

73 Space in which this family's regions live. 

74 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

75 The members of this family, ordered according to the priority used 

76 in `choose` (first-choice member first). 

77 """ 

78 

79 def __init__( 

80 self, 

81 name: str, 

82 space: TopologicalSpace, 

83 *, 

84 members: NamedValueAbstractSet[DimensionElement], 

85 ): 

86 super().__init__(name, space) 

87 self.members = members 

88 

89 def choose(self, endpoints: Set[str], universe: DimensionUniverse) -> DimensionElement: 

90 # Docstring inherited from TopologicalFamily. 

91 for member in self.members: 

92 if member.name in endpoints: 

93 return member 

94 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

95 

96 @property 

97 @cached_getter 

98 def governor(self) -> GovernorDimension: 

99 """Return `GovernorDimension` common to all members of this family. 

100 

101 (`GovernorDimension`). 

102 """ 

103 governors = {m.governor for m in self.members} 

104 if None in governors: 

105 raise RuntimeError( 

106 f"Bad {self.space.name} family definition {self.name}: at least one member " 

107 f"in {self.members} has no GovernorDimension dependency." 

108 ) 

109 try: 

110 (result,) = governors 

111 except ValueError: 

112 raise RuntimeError( 

113 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

114 f"in {self.members}." 

115 ) from None 

116 return result # type: ignore 

117 

118 members: NamedValueAbstractSet[DimensionElement] 

119 """The members of this family, ordered according to the priority used in 

120 `choose` (first-choice member first). 

121 """ 

122 

123 

124class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

125 """A construction visitor for `DatabaseTopologicalFamily`. 

126 

127 This visitor depends on (and is thus visited after) its members. 

128 

129 Parameters 

130 ---------- 

131 name : `str` 

132 Name of the family. 

133 members : `~collections.abc.Iterable` [ `str` ] 

134 The names of the members of this family, ordered according to the 

135 priority used in `choose` (first-choice member first). 

136 """ 

137 

138 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

139 super().__init__(name) 

140 self._space = space 

141 self._members = tuple(members) 

142 

143 def hasDependenciesIn(self, others: Set[str]) -> bool: 

144 # Docstring inherited from DimensionConstructionVisitor. 

145 return not others.isdisjoint(self._members) 

146 

147 def visit(self, builder: DimensionConstructionBuilder) -> None: 

148 # Docstring inherited from DimensionConstructionVisitor. 

149 members = NamedValueSet(builder.elements[name] for name in self._members) 

150 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

151 builder.topology[self._space].add(family) 

152 for member in members: 

153 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination) 

154 other = member._topology.setdefault(self._space, family) 

155 if other is not family: 

156 raise RuntimeError( 

157 f"{member.name} is declared to be a member of (at least) two " 

158 f"{self._space.name} families: {other.name} and {family.name}." 

159 ) 

160 

161 

162class DatabaseDimensionElement(DimensionElement): 

163 """An intermediate base class for `DimensionElement` database classes. 

164 

165 Theese classes are ones whose instances map directly to a database 

166 table or query. 

167 

168 Parameters 

169 ---------- 

170 name : `str` 

171 Name of the dimension. 

172 storage : `dict` 

173 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

174 that will back this element in the registry (in a "cls" key) along 

175 with any other construction keyword arguments (in other keys). 

176 implied : `NamedValueAbstractSet` [ `Dimension` ] 

177 Other dimensions whose keys are included in this dimension's (logical) 

178 table as foreign keys. 

179 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

180 Field specifications for all non-key fields in this dimension's table. 

181 """ 

182 

183 def __init__( 

184 self, 

185 name: str, 

186 storage: dict, 

187 *, 

188 implied: NamedValueAbstractSet[Dimension], 

189 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

190 ): 

191 self._name = name 

192 self._storage = storage 

193 self._implied = implied 

194 self._metadata = metadata 

195 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

196 

197 @property 

198 def name(self) -> str: 

199 # Docstring inherited from TopologicalRelationshipEndpoint. 

200 return self._name 

201 

202 @property 

203 def implied(self) -> NamedValueAbstractSet[Dimension]: 

204 # Docstring inherited from DimensionElement. 

205 return self._implied 

206 

207 @property 

208 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

209 # Docstring inherited from DimensionElement. 

210 return self._metadata 

211 

212 @property 

213 def viewOf(self) -> str | None: 

214 # Docstring inherited from DimensionElement. 

215 # This is a bit encapsulation-breaking; these storage config values 

216 # are supposed to be opaque here, and just forwarded on to some 

217 # DimensionRecordStorage implementation. The long-term fix is to 

218 # move viewOf entirely, by changing the code that relies on it to rely 

219 # on the DimensionRecordStorage object instead. 

220 storage = self._storage.get("nested") 

221 if storage is None: 

222 storage = self._storage 

223 return storage.get("view_of") 

224 

225 @property 

226 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

227 # Docstring inherited from TopologicalRelationshipEndpoint 

228 return MappingProxyType(self._topology) 

229 

230 @property 

231 def spatial(self) -> DatabaseTopologicalFamily | None: 

232 # Docstring inherited from TopologicalRelationshipEndpoint 

233 return self.topology.get(TopologicalSpace.SPATIAL) 

234 

235 @property 

236 def temporal(self) -> DatabaseTopologicalFamily | None: 

237 # Docstring inherited from TopologicalRelationshipEndpoint 

238 return self.topology.get(TopologicalSpace.TEMPORAL) 

239 

240 def makeStorage( 

241 self, 

242 db: Database, 

243 *, 

244 context: StaticTablesContext | None = None, 

245 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

246 view_target: DatabaseDimensionRecordStorage | None = None, 

247 ) -> DatabaseDimensionRecordStorage: 

248 """Make the dimension record storage instance for this database. 

249 

250 Constructs the `DimensionRecordStorage` instance that should 

251 be used to back this element in a registry. 

252 

253 Parameters 

254 ---------- 

255 db : `Database` 

256 Interface to the underlying database engine and namespace. 

257 context : `StaticTablesContext`, optional 

258 If provided, an object to use to create any new tables. If not 

259 provided, ``db.ensureTableExists`` should be used instead. 

260 governors : `NamedKeyMapping` 

261 Mapping from `GovernorDimension` to the record storage backend for 

262 that dimension, containing all governor dimensions. 

263 view_target : `DatabaseDimensionRecordStorage`, optional 

264 Storage object for the element this target's storage is a view of 

265 (i.e. when `viewOf` is not `None`). 

266 

267 Returns 

268 ------- 

269 storage : `DatabaseDimensionRecordStorage` 

270 Storage object that should back this element in a registry. 

271 """ 

272 from ..registry.interfaces import DatabaseDimensionRecordStorage 

273 

274 cls = doImportType(self._storage["cls"]) 

275 assert issubclass(cls, DatabaseDimensionRecordStorage) 

276 return cls.initialize( 

277 db, 

278 self, 

279 context=context, 

280 config=self._storage, 

281 governors=governors, 

282 view_target=view_target, 

283 ) 

284 

285 

286class DatabaseDimension(Dimension, DatabaseDimensionElement): 

287 """A `Dimension` class that maps directly to a database table or query. 

288 

289 Parameters 

290 ---------- 

291 name : `str` 

292 Name of the dimension. 

293 storage : `dict` 

294 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

295 that will back this element in the registry (in a "cls" key) along 

296 with any other construction keyword arguments (in other keys). 

297 required : `NamedValueSet` [ `Dimension` ] 

298 Other dimensions whose keys are part of the compound primary key for 

299 this dimension's (logical) table, as well as references to their own 

300 tables. The ``required`` parameter does not include ``self`` (it 

301 can't, of course), but the corresponding attribute does - it is added 

302 by the constructor. 

303 implied : `NamedValueAbstractSet` [ `Dimension` ] 

304 Other dimensions whose keys are included in this dimension's (logical) 

305 table as foreign keys. 

306 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

307 Field specifications for all non-key fields in this dimension's table. 

308 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

309 Fields that can each be used to uniquely identify this dimension (given 

310 values for all required dimensions). The first of these is used as 

311 (part of) this dimension's table's primary key, while others are used 

312 to define unique constraints. 

313 

314 Notes 

315 ----- 

316 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

317 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

318 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

319 members. 

320 """ 

321 

322 def __init__( 

323 self, 

324 name: str, 

325 storage: dict, 

326 *, 

327 required: NamedValueSet[Dimension], 

328 implied: NamedValueAbstractSet[Dimension], 

329 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

330 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

331 ): 

332 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

333 required.add(self) 

334 self._required = required.freeze() 

335 self._uniqueKeys = uniqueKeys 

336 

337 @property 

338 def required(self) -> NamedValueAbstractSet[Dimension]: 

339 # Docstring inherited from DimensionElement. 

340 return self._required 

341 

342 @property 

343 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

344 # Docstring inherited from Dimension. 

345 return self._uniqueKeys 

346 

347 

348class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

349 """A combination class that maps directly to a database table or query. 

350 

351 Parameters 

352 ---------- 

353 name : `str` 

354 Name of the dimension. 

355 storage : `dict` 

356 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

357 that will back this element in the registry (in a "cls" key) along 

358 with any other construction keyword arguments (in other keys). 

359 required : `NamedValueAbstractSet` [ `Dimension` ] 

360 Dimensions whose keys define the compound primary key for this 

361 combinations's (logical) table, as well as references to their own 

362 tables. 

363 implied : `NamedValueAbstractSet` [ `Dimension` ] 

364 Dimensions whose keys are included in this combinations's (logical) 

365 table as foreign keys. 

366 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

367 Field specifications for all non-key fields in this combination's 

368 table. 

369 alwaysJoin : `bool`, optional 

370 If `True`, always include this element in any query or data ID in 

371 which its ``required`` dimensions appear, because it defines a 

372 relationship between those dimensions that must always be satisfied. 

373 

374 Notes 

375 ----- 

376 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

377 but it is the responsibility of 

378 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

379 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

380 

381 This class has a lot in common with `DatabaseDimension`, but they are 

382 expected to diverge in future changes, and the only way to make them share 

383 method implementations would be via multiple inheritance. Given the 

384 trivial nature of all of those implementations, this does not seem worth 

385 the drawbacks (particularly the constraints it imposes on constructor 

386 signatures). 

387 """ 

388 

389 def __init__( 

390 self, 

391 name: str, 

392 storage: dict, 

393 *, 

394 required: NamedValueAbstractSet[Dimension], 

395 implied: NamedValueAbstractSet[Dimension], 

396 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

397 alwaysJoin: bool, 

398 populated_by: Dimension | None, 

399 ): 

400 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

401 self._required = required 

402 self._alwaysJoin = alwaysJoin 

403 self._populated_by = populated_by 

404 

405 @property 

406 def required(self) -> NamedValueAbstractSet[Dimension]: 

407 # Docstring inherited from DimensionElement. 

408 return self._required 

409 

410 @property 

411 def alwaysJoin(self) -> bool: 

412 # Docstring inherited from DimensionElement. 

413 return self._alwaysJoin 

414 

415 @property 

416 def populated_by(self) -> Dimension | None: 

417 # Docstring inherited. 

418 return self._populated_by 

419 

420 

421class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

422 """Construction visitor for database dimension and dimension combination. 

423 

424 Specifically, a construction visitor for `DatabaseDimension` and 

425 `DatabaseDimensionCombination`. 

426 

427 Parameters 

428 ---------- 

429 name : `str` 

430 Name of the dimension. 

431 storage : `dict` 

432 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

433 that will back this element in the registry (in a "cls" key) along 

434 with any other construction keyword arguments (in other keys). 

435 required : `~collections.abc.Set` [ `Dimension` ] 

436 Names of dimensions whose keys define the compound primary key for this 

437 element's (logical) table, as well as references to their own 

438 tables. 

439 implied : `~collections.abc.Set` [ `Dimension` ] 

440 Names of dimension whose keys are included in this elements's 

441 (logical) table as foreign keys. 

442 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

443 Field specifications for all non-key fields in this element's table. 

444 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

445 Fields that can each be used to uniquely identify this dimension (given 

446 values for all required dimensions). The first of these is used as 

447 (part of) this dimension's table's primary key, while others are used 

448 to define unique constraints. Should be empty for 

449 `DatabaseDimensionCombination` definitions. 

450 alwaysJoin : `bool`, optional 

451 If `True`, always include this element in any query or data ID in 

452 which its ``required`` dimensions appear, because it defines a 

453 relationship between those dimensions that must always be satisfied. 

454 Should only be provided when a `DimensionCombination` is being 

455 constructed. 

456 populated_by: `Dimension`, optional 

457 The dimension that this element's records are always inserted, 

458 exported, and imported alongside. 

459 """ 

460 

461 def __init__( 

462 self, 

463 name: str, 

464 storage: dict, 

465 required: set[str], 

466 implied: set[str], 

467 metadata: Iterable[ddl.FieldSpec] = (), 

468 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

469 alwaysJoin: bool = False, 

470 populated_by: str | None = None, 

471 ): 

472 super().__init__(name) 

473 self._storage = storage 

474 self._required = required 

475 self._implied = implied 

476 self._metadata = NamedValueSet(metadata).freeze() 

477 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

478 self._alwaysJoin = alwaysJoin 

479 self._populated_by = populated_by 

480 

481 def hasDependenciesIn(self, others: Set[str]) -> bool: 

482 # Docstring inherited from DimensionConstructionVisitor. 

483 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

484 

485 def visit(self, builder: DimensionConstructionBuilder) -> None: 

486 # Docstring inherited from DimensionConstructionVisitor. 

487 # Expand required dependencies. 

488 for name in tuple(self._required): # iterate over copy 

489 self._required.update(builder.dimensions[name].required.names) 

490 # Transform required and implied Dimension names into instances, 

491 # and reorder to match builder's order. 

492 required: NamedValueSet[Dimension] = NamedValueSet() 

493 implied: NamedValueSet[Dimension] = NamedValueSet() 

494 for dimension in builder.dimensions: 

495 if dimension.name in self._required: 

496 required.add(dimension) 

497 if dimension.name in self._implied: 

498 implied.add(dimension) 

499 

500 if self._uniqueKeys: 

501 if self._alwaysJoin: 

502 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

503 # Special handling for creating Dimension instances. 

504 dimension = DatabaseDimension( 

505 self.name, 

506 storage=self._storage, 

507 required=required, 

508 implied=implied.freeze(), 

509 metadata=self._metadata, 

510 uniqueKeys=self._uniqueKeys, 

511 ) 

512 builder.dimensions.add(dimension) 

513 builder.elements.add(dimension) 

514 else: 

515 # Special handling for creating DimensionCombination instances. 

516 combination = DatabaseDimensionCombination( 

517 self.name, 

518 storage=self._storage, 

519 required=required, 

520 implied=implied.freeze(), 

521 metadata=self._metadata, 

522 alwaysJoin=self._alwaysJoin, 

523 populated_by=( 

524 builder.dimensions[self._populated_by] if self._populated_by is not None else None 

525 ), 

526 ) 

527 builder.elements.add(combination)