Coverage for python/lsst/daf/butler/dimensions/_database.py: 44%

143 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "DatabaseDimension", 

32 "DatabaseDimensionCombination", 

33 "DatabaseDimensionElement", 

34 "DatabaseTopologicalFamily", 

35) 

36 

37from collections.abc import Iterable, Mapping, Set 

38from types import MappingProxyType 

39from typing import TYPE_CHECKING 

40 

41from lsst.utils import doImportType 

42from lsst.utils.classes import cached_getter 

43 

44from .. import ddl 

45from .._named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

46from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

47from ._elements import Dimension, DimensionCombination, DimensionElement 

48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

49 

50if TYPE_CHECKING: 

51 from ..registry.interfaces import ( 

52 Database, 

53 DatabaseDimensionRecordStorage, 

54 GovernorDimensionRecordStorage, 

55 StaticTablesContext, 

56 ) 

57 from ._governor import GovernorDimension 

58 

59 

60class DatabaseTopologicalFamily(TopologicalFamily): 

61 """Database topological family implementation. 

62 

63 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

64 `DatabaseDimensionCombination` objects that have direct database 

65 representations. 

66 

67 Parameters 

68 ---------- 

69 name : `str` 

70 Name of the family. 

71 space : `TopologicalSpace` 

72 Space in which this family's regions live. 

73 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

74 The members of this family, ordered according to the priority used 

75 in `choose` (first-choice member first). 

76 """ 

77 

78 def __init__( 

79 self, 

80 name: str, 

81 space: TopologicalSpace, 

82 *, 

83 members: NamedValueAbstractSet[DimensionElement], 

84 ): 

85 super().__init__(name, space) 

86 self.members = members 

87 

88 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

89 # Docstring inherited from TopologicalFamily. 

90 for member in self.members: 

91 if member in endpoints: 

92 return member 

93 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

94 

95 @property 

96 @cached_getter 

97 def governor(self) -> GovernorDimension: 

98 """Return `GovernorDimension` common to all members of this family. 

99 

100 (`GovernorDimension`). 

101 """ 

102 governors = {m.governor for m in self.members} 

103 if None in governors: 

104 raise RuntimeError( 

105 f"Bad {self.space.name} family definition {self.name}: at least one member " 

106 f"in {self.members} has no GovernorDimension dependency." 

107 ) 

108 try: 

109 (result,) = governors 

110 except ValueError: 

111 raise RuntimeError( 

112 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

113 f"in {self.members}." 

114 ) from None 

115 return result # type: ignore 

116 

117 members: NamedValueAbstractSet[DimensionElement] 

118 """The members of this family, ordered according to the priority used in 

119 `choose` (first-choice member first). 

120 """ 

121 

122 

123class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

124 """A construction visitor for `DatabaseTopologicalFamily`. 

125 

126 This visitor depends on (and is thus visited after) its members. 

127 

128 Parameters 

129 ---------- 

130 name : `str` 

131 Name of the family. 

132 members : `~collections.abc.Iterable` [ `str` ] 

133 The names of the members of this family, ordered according to the 

134 priority used in `choose` (first-choice member first). 

135 """ 

136 

137 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

138 super().__init__(name) 

139 self._space = space 

140 self._members = tuple(members) 

141 

142 def hasDependenciesIn(self, others: Set[str]) -> bool: 

143 # Docstring inherited from DimensionConstructionVisitor. 

144 return not others.isdisjoint(self._members) 

145 

146 def visit(self, builder: DimensionConstructionBuilder) -> None: 

147 # Docstring inherited from DimensionConstructionVisitor. 

148 members = NamedValueSet(builder.elements[name] for name in self._members) 

149 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

150 builder.topology[self._space].add(family) 

151 for member in members: 

152 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination) 

153 other = member._topology.setdefault(self._space, family) 

154 if other is not family: 

155 raise RuntimeError( 

156 f"{member.name} is declared to be a member of (at least) two " 

157 f"{self._space.name} families: {other.name} and {family.name}." 

158 ) 

159 

160 

161class DatabaseDimensionElement(DimensionElement): 

162 """An intermediate base class for `DimensionElement` database classes. 

163 

164 Theese classes are ones whose instances map directly to a database 

165 table or query. 

166 

167 Parameters 

168 ---------- 

169 name : `str` 

170 Name of the dimension. 

171 storage : `dict` 

172 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

173 that will back this element in the registry (in a "cls" key) along 

174 with any other construction keyword arguments (in other keys). 

175 implied : `NamedValueAbstractSet` [ `Dimension` ] 

176 Other dimensions whose keys are included in this dimension's (logical) 

177 table as foreign keys. 

178 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

179 Field specifications for all non-key fields in this dimension's table. 

180 """ 

181 

182 def __init__( 

183 self, 

184 name: str, 

185 storage: dict, 

186 *, 

187 implied: NamedValueAbstractSet[Dimension], 

188 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

189 ): 

190 self._name = name 

191 self._storage = storage 

192 self._implied = implied 

193 self._metadata = metadata 

194 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

195 

196 @property 

197 def name(self) -> str: 

198 # Docstring inherited from TopologicalRelationshipEndpoint. 

199 return self._name 

200 

201 @property 

202 def implied(self) -> NamedValueAbstractSet[Dimension]: 

203 # Docstring inherited from DimensionElement. 

204 return self._implied 

205 

206 @property 

207 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

208 # Docstring inherited from DimensionElement. 

209 return self._metadata 

210 

211 @property 

212 def viewOf(self) -> str | None: 

213 # Docstring inherited from DimensionElement. 

214 # This is a bit encapsulation-breaking; these storage config values 

215 # are supposed to be opaque here, and just forwarded on to some 

216 # DimensionRecordStorage implementation. The long-term fix is to 

217 # move viewOf entirely, by changing the code that relies on it to rely 

218 # on the DimensionRecordStorage object instead. 

219 storage = self._storage.get("nested") 

220 if storage is None: 

221 storage = self._storage 

222 return storage.get("view_of") 

223 

224 @property 

225 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

226 # Docstring inherited from TopologicalRelationshipEndpoint 

227 return MappingProxyType(self._topology) 

228 

229 @property 

230 def spatial(self) -> DatabaseTopologicalFamily | None: 

231 # Docstring inherited from TopologicalRelationshipEndpoint 

232 return self.topology.get(TopologicalSpace.SPATIAL) 

233 

234 @property 

235 def temporal(self) -> DatabaseTopologicalFamily | None: 

236 # Docstring inherited from TopologicalRelationshipEndpoint 

237 return self.topology.get(TopologicalSpace.TEMPORAL) 

238 

239 def makeStorage( 

240 self, 

241 db: Database, 

242 *, 

243 context: StaticTablesContext | None = None, 

244 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

245 view_target: DatabaseDimensionRecordStorage | None = None, 

246 ) -> DatabaseDimensionRecordStorage: 

247 """Make the dimension record storage instance for this database. 

248 

249 Constructs the `DimensionRecordStorage` instance that should 

250 be used to back this element in a registry. 

251 

252 Parameters 

253 ---------- 

254 db : `Database` 

255 Interface to the underlying database engine and namespace. 

256 context : `StaticTablesContext`, optional 

257 If provided, an object to use to create any new tables. If not 

258 provided, ``db.ensureTableExists`` should be used instead. 

259 governors : `NamedKeyMapping` 

260 Mapping from `GovernorDimension` to the record storage backend for 

261 that dimension, containing all governor dimensions. 

262 view_target : `DatabaseDimensionRecordStorage`, optional 

263 Storage object for the element this target's storage is a view of 

264 (i.e. when `viewOf` is not `None`). 

265 

266 Returns 

267 ------- 

268 storage : `DatabaseDimensionRecordStorage` 

269 Storage object that should back this element in a registry. 

270 """ 

271 from ..registry.interfaces import DatabaseDimensionRecordStorage 

272 

273 cls = doImportType(self._storage["cls"]) 

274 assert issubclass(cls, DatabaseDimensionRecordStorage) 

275 return cls.initialize( 

276 db, 

277 self, 

278 context=context, 

279 config=self._storage, 

280 governors=governors, 

281 view_target=view_target, 

282 ) 

283 

284 

285class DatabaseDimension(Dimension, DatabaseDimensionElement): 

286 """A `Dimension` class that maps directly to a database table or query. 

287 

288 Parameters 

289 ---------- 

290 name : `str` 

291 Name of the dimension. 

292 storage : `dict` 

293 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

294 that will back this element in the registry (in a "cls" key) along 

295 with any other construction keyword arguments (in other keys). 

296 required : `NamedValueSet` [ `Dimension` ] 

297 Other dimensions whose keys are part of the compound primary key for 

298 this dimension's (logical) table, as well as references to their own 

299 tables. The ``required`` parameter does not include ``self`` (it 

300 can't, of course), but the corresponding attribute does - it is added 

301 by the constructor. 

302 implied : `NamedValueAbstractSet` [ `Dimension` ] 

303 Other dimensions whose keys are included in this dimension's (logical) 

304 table as foreign keys. 

305 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

306 Field specifications for all non-key fields in this dimension's table. 

307 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

308 Fields that can each be used to uniquely identify this dimension (given 

309 values for all required dimensions). The first of these is used as 

310 (part of) this dimension's table's primary key, while others are used 

311 to define unique constraints. 

312 

313 Notes 

314 ----- 

315 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

316 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

317 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

318 members. 

319 """ 

320 

321 def __init__( 

322 self, 

323 name: str, 

324 storage: dict, 

325 *, 

326 required: NamedValueSet[Dimension], 

327 implied: NamedValueAbstractSet[Dimension], 

328 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

329 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

330 ): 

331 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

332 required.add(self) 

333 self._required = required.freeze() 

334 self._uniqueKeys = uniqueKeys 

335 

336 @property 

337 def required(self) -> NamedValueAbstractSet[Dimension]: 

338 # Docstring inherited from DimensionElement. 

339 return self._required 

340 

341 @property 

342 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

343 # Docstring inherited from Dimension. 

344 return self._uniqueKeys 

345 

346 

347class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

348 """A combination class that maps directly to a database table or query. 

349 

350 Parameters 

351 ---------- 

352 name : `str` 

353 Name of the dimension. 

354 storage : `dict` 

355 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

356 that will back this element in the registry (in a "cls" key) along 

357 with any other construction keyword arguments (in other keys). 

358 required : `NamedValueAbstractSet` [ `Dimension` ] 

359 Dimensions whose keys define the compound primary key for this 

360 combinations's (logical) table, as well as references to their own 

361 tables. 

362 implied : `NamedValueAbstractSet` [ `Dimension` ] 

363 Dimensions whose keys are included in this combinations's (logical) 

364 table as foreign keys. 

365 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

366 Field specifications for all non-key fields in this combination's 

367 table. 

368 alwaysJoin : `bool`, optional 

369 If `True`, always include this element in any query or data ID in 

370 which its ``required`` dimensions appear, because it defines a 

371 relationship between those dimensions that must always be satisfied. 

372 

373 Notes 

374 ----- 

375 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

376 but it is the responsibility of 

377 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

378 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

379 

380 This class has a lot in common with `DatabaseDimension`, but they are 

381 expected to diverge in future changes, and the only way to make them share 

382 method implementations would be via multiple inheritance. Given the 

383 trivial nature of all of those implementations, this does not seem worth 

384 the drawbacks (particularly the constraints it imposes on constructor 

385 signatures). 

386 """ 

387 

388 def __init__( 

389 self, 

390 name: str, 

391 storage: dict, 

392 *, 

393 required: NamedValueAbstractSet[Dimension], 

394 implied: NamedValueAbstractSet[Dimension], 

395 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

396 alwaysJoin: bool, 

397 populated_by: Dimension | None, 

398 ): 

399 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

400 self._required = required 

401 self._alwaysJoin = alwaysJoin 

402 self._populated_by = populated_by 

403 

404 @property 

405 def required(self) -> NamedValueAbstractSet[Dimension]: 

406 # Docstring inherited from DimensionElement. 

407 return self._required 

408 

409 @property 

410 def alwaysJoin(self) -> bool: 

411 # Docstring inherited from DimensionElement. 

412 return self._alwaysJoin 

413 

414 @property 

415 def populated_by(self) -> Dimension | None: 

416 # Docstring inherited. 

417 return self._populated_by 

418 

419 

420class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

421 """Construction visitor for database dimension and dimension combination. 

422 

423 Specifically, a construction visitor for `DatabaseDimension` and 

424 `DatabaseDimensionCombination`. 

425 

426 Parameters 

427 ---------- 

428 name : `str` 

429 Name of the dimension. 

430 storage : `dict` 

431 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

432 that will back this element in the registry (in a "cls" key) along 

433 with any other construction keyword arguments (in other keys). 

434 required : `~collections.abc.Set` [ `Dimension` ] 

435 Names of dimensions whose keys define the compound primary key for this 

436 element's (logical) table, as well as references to their own 

437 tables. 

438 implied : `~collections.abc.Set` [ `Dimension` ] 

439 Names of dimension whose keys are included in this elements's 

440 (logical) table as foreign keys. 

441 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

442 Field specifications for all non-key fields in this element's table. 

443 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

444 Fields that can each be used to uniquely identify this dimension (given 

445 values for all required dimensions). The first of these is used as 

446 (part of) this dimension's table's primary key, while others are used 

447 to define unique constraints. Should be empty for 

448 `DatabaseDimensionCombination` definitions. 

449 alwaysJoin : `bool`, optional 

450 If `True`, always include this element in any query or data ID in 

451 which its ``required`` dimensions appear, because it defines a 

452 relationship between those dimensions that must always be satisfied. 

453 Should only be provided when a `DimensionCombination` is being 

454 constructed. 

455 populated_by: `Dimension`, optional 

456 The dimension that this element's records are always inserted, 

457 exported, and imported alongside. 

458 """ 

459 

460 def __init__( 

461 self, 

462 name: str, 

463 storage: dict, 

464 required: set[str], 

465 implied: set[str], 

466 metadata: Iterable[ddl.FieldSpec] = (), 

467 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

468 alwaysJoin: bool = False, 

469 populated_by: str | None = None, 

470 ): 

471 super().__init__(name) 

472 self._storage = storage 

473 self._required = required 

474 self._implied = implied 

475 self._metadata = NamedValueSet(metadata).freeze() 

476 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

477 self._alwaysJoin = alwaysJoin 

478 self._populated_by = populated_by 

479 

480 def hasDependenciesIn(self, others: Set[str]) -> bool: 

481 # Docstring inherited from DimensionConstructionVisitor. 

482 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

483 

484 def visit(self, builder: DimensionConstructionBuilder) -> None: 

485 # Docstring inherited from DimensionConstructionVisitor. 

486 # Expand required dependencies. 

487 for name in tuple(self._required): # iterate over copy 

488 self._required.update(builder.dimensions[name].required.names) 

489 # Transform required and implied Dimension names into instances, 

490 # and reorder to match builder's order. 

491 required: NamedValueSet[Dimension] = NamedValueSet() 

492 implied: NamedValueSet[Dimension] = NamedValueSet() 

493 for dimension in builder.dimensions: 

494 if dimension.name in self._required: 

495 required.add(dimension) 

496 if dimension.name in self._implied: 

497 implied.add(dimension) 

498 

499 if self._uniqueKeys: 

500 if self._alwaysJoin: 

501 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

502 # Special handling for creating Dimension instances. 

503 dimension = DatabaseDimension( 

504 self.name, 

505 storage=self._storage, 

506 required=required, 

507 implied=implied.freeze(), 

508 metadata=self._metadata, 

509 uniqueKeys=self._uniqueKeys, 

510 ) 

511 builder.dimensions.add(dimension) 

512 builder.elements.add(dimension) 

513 else: 

514 # Special handling for creating DimensionCombination instances. 

515 combination = DatabaseDimensionCombination( 

516 self.name, 

517 storage=self._storage, 

518 required=required, 

519 implied=implied.freeze(), 

520 metadata=self._metadata, 

521 alwaysJoin=self._alwaysJoin, 

522 populated_by=( 

523 builder.dimensions[self._populated_by] if self._populated_by is not None else None 

524 ), 

525 ) 

526 builder.elements.add(combination)