Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from types import MappingProxyType 

32from typing import ( 

33 AbstractSet, 

34 Dict, 

35 Iterable, 

36 Mapping, 

37 Optional, 

38 Set, 

39 TYPE_CHECKING, 

40) 

41 

42from lsst.utils import doImport 

43 

44from .. import ddl 

45from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

46from ..utils import cached_getter 

47from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

48 

49from ._elements import Dimension, DimensionCombination, DimensionElement 

50from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

51 

52if TYPE_CHECKING: 52 ↛ 53line 52 didn't jump to line 53, because the condition on line 52 was never true

53 from ._governor import GovernorDimension 

54 from ...registry.interfaces import ( 

55 Database, 

56 DatabaseDimensionRecordStorage, 

57 GovernorDimensionRecordStorage, 

58 StaticTablesContext, 

59 ) 

60 

61 

62class DatabaseTopologicalFamily(TopologicalFamily): 

63 """Database topological family implementation. 

64 

65 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

66 `DatabaseDimensionCombination` objects that have direct database 

67 representations. 

68 

69 Parameters 

70 ---------- 

71 name : `str` 

72 Name of the family. 

73 space : `TopologicalSpace` 

74 Space in which this family's regions live. 

75 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

76 The members of this family, ordered according to the priority used 

77 in `choose` (first-choice member first). 

78 """ 

79 

80 def __init__( 

81 self, 

82 name: str, 

83 space: TopologicalSpace, *, 

84 members: NamedValueAbstractSet[DimensionElement], 

85 ): 

86 super().__init__(name, space) 

87 self.members = members 

88 

89 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

90 # Docstring inherited from TopologicalFamily. 

91 for member in self.members: 

92 if member in endpoints: 

93 return member 

94 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

95 

96 @property # type: ignore 

97 @cached_getter 

98 def governor(self) -> GovernorDimension: 

99 """Return `GovernorDimension` common to all members of this family. 

100 

101 (`GovernorDimension`). 

102 """ 

103 governors = set(m.governor for m in self.members) 

104 if None in governors: 

105 raise RuntimeError( 

106 f"Bad {self.space.name} family definition {self.name}: at least one member " 

107 f"in {self.members} has no GovernorDimension dependency." 

108 ) 

109 try: 

110 (result,) = governors 

111 except ValueError: 

112 raise RuntimeError( 

113 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

114 f"in {self.members}." 

115 ) from None 

116 return result # type: ignore 

117 

118 members: NamedValueAbstractSet[DimensionElement] 

119 """The members of this family, ordered according to the priority used in 

120 `choose` (first-choice member first). 

121 """ 

122 

123 

124class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

125 """A construction visitor for `DatabaseTopologicalFamily`. 

126 

127 This visitor depends on (and is thus visited after) its members. 

128 

129 Parameters 

130 ---------- 

131 name : `str` 

132 Name of the family. 

133 members : `Iterable` [ `str` ] 

134 The names of the members of this family, ordered according to the 

135 priority used in `choose` (first-choice member first). 

136 """ 

137 

138 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

139 super().__init__(name) 

140 self._space = space 

141 self._members = tuple(members) 

142 

143 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

144 # Docstring inherited from DimensionConstructionVisitor. 

145 return not others.isdisjoint(self._members) 

146 

147 def visit(self, builder: DimensionConstructionBuilder) -> None: 

148 # Docstring inherited from DimensionConstructionVisitor. 

149 members = NamedValueSet(builder.elements[name] for name in self._members) 

150 family = DatabaseTopologicalFamily( 

151 self.name, 

152 self._space, 

153 members=members.freeze() 

154 ) 

155 builder.topology[self._space].add(family) 

156 for member in members: 

157 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination)) 

158 other = member._topology.setdefault(self._space, family) 

159 if other is not family: 

160 raise RuntimeError(f"{member.name} is declared to be a member of (at least) two " 

161 f"{self._space.name} families: {other.name} and {family.name}.") 

162 

163 

164class DatabaseDimensionElement(DimensionElement): 

165 """An intermediate base class for `DimensionElement` database classes. 

166 

167 Theese classes are ones whose instances map directly to a database 

168 table or query. 

169 

170 Parameters 

171 ---------- 

172 name : `str` 

173 Name of the dimension. 

174 storage : `dict` 

175 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

176 that will back this element in the registry (in a "cls" key) along 

177 with any other construction keyword arguments (in other keys). 

178 implied : `NamedValueAbstractSet` [ `Dimension` ] 

179 Other dimensions whose keys are included in this dimension's (logical) 

180 table as foreign keys. 

181 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

182 Field specifications for all non-key fields in this dimension's table. 

183 """ 

184 

185 def __init__( 

186 self, 

187 name: str, 

188 storage: dict, *, 

189 implied: NamedValueAbstractSet[Dimension], 

190 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

191 ): 

192 self._name = name 

193 self._storage = storage 

194 self._implied = implied 

195 self._metadata = metadata 

196 self._topology: Dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

197 

198 @property 

199 def name(self) -> str: 

200 # Docstring inherited from TopoogicalRelationshipEndpoint. 

201 return self._name 

202 

203 @property 

204 def implied(self) -> NamedValueAbstractSet[Dimension]: 

205 # Docstring inherited from DimensionElement. 

206 return self._implied 

207 

208 @property 

209 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

210 # Docstring inherited from DimensionElement. 

211 return self._metadata 

212 

213 @property 

214 def viewOf(self) -> Optional[str]: 

215 # Docstring inherited from DimensionElement. 

216 # This is a bit encapsulation-breaking; these storage config values 

217 # are supposed to be opaque here, and just forwarded on to some 

218 # DimensionRecordStorage implementation. The long-term fix is to 

219 # move viewOf entirely, by changing the code that relies on it to rely 

220 # on the DimensionRecordStorage object instead. 

221 storage = self._storage.get("nested") 

222 if storage is None: 

223 storage = self._storage 

224 return storage.get("view_of") 

225 

226 @property 

227 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

228 # Docstring inherited from TopologicalRelationshipEndpoint 

229 return MappingProxyType(self._topology) 

230 

231 @property 

232 def spatial(self) -> Optional[DatabaseTopologicalFamily]: 

233 # Docstring inherited from TopologicalRelationshipEndpoint 

234 return self.topology.get(TopologicalSpace.SPATIAL) 

235 

236 @property 

237 def temporal(self) -> Optional[DatabaseTopologicalFamily]: 

238 # Docstring inherited from TopologicalRelationshipEndpoint 

239 return self.topology.get(TopologicalSpace.TEMPORAL) 

240 

241 def makeStorage( 

242 self, 

243 db: Database, *, 

244 context: Optional[StaticTablesContext] = None, 

245 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

246 ) -> DatabaseDimensionRecordStorage: 

247 """Make the dimension record storage instance for this database. 

248 

249 Constructs the `DimensionRecordStorage` instance that should 

250 be used to back this element in a registry. 

251 

252 Parameters 

253 ---------- 

254 db : `Database` 

255 Interface to the underlying database engine and namespace. 

256 context : `StaticTablesContext`, optional 

257 If provided, an object to use to create any new tables. If not 

258 provided, ``db.ensureTableExists`` should be used instead. 

259 governors : `NamedKeyMapping` 

260 Mapping from `GovernorDimension` to the record storage backend for 

261 that dimension, containing all governor dimensions. 

262 

263 Returns 

264 ------- 

265 storage : `DatabaseDimensionRecordStorage` 

266 Storage object that should back this element in a registry. 

267 """ 

268 from ...registry.interfaces import DatabaseDimensionRecordStorage 

269 cls = doImport(self._storage["cls"]) 

270 assert issubclass(cls, DatabaseDimensionRecordStorage) 

271 return cls.initialize(db, self, context=context, config=self._storage, governors=governors) 

272 

273 

274class DatabaseDimension(Dimension, DatabaseDimensionElement): 

275 """A `Dimension` class that maps directly to a database table or query. 

276 

277 Parameters 

278 ---------- 

279 name : `str` 

280 Name of the dimension. 

281 storage : `dict` 

282 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

283 that will back this element in the registry (in a "cls" key) along 

284 with any other construction keyword arguments (in other keys). 

285 required : `NamedValueSet` [ `Dimension` ] 

286 Other dimensions whose keys are part of the compound primary key for 

287 this dimension's (logical) table, as well as references to their own 

288 tables. The ``required`` parameter does not include ``self`` (it 

289 can't, of course), but the corresponding attribute does - it is added 

290 by the constructor. 

291 implied : `NamedValueAbstractSet` [ `Dimension` ] 

292 Other dimensions whose keys are included in this dimension's (logical) 

293 table as foreign keys. 

294 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

295 Field specifications for all non-key fields in this dimension's table. 

296 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

297 Fields that can each be used to uniquely identify this dimension (given 

298 values for all required dimensions). The first of these is used as 

299 (part of) this dimension's table's primary key, while others are used 

300 to define unique constraints. 

301 

302 Notes 

303 ----- 

304 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

305 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

306 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

307 members. 

308 """ 

309 

310 def __init__( 

311 self, 

312 name: str, 

313 storage: dict, *, 

314 required: NamedValueSet[Dimension], 

315 implied: NamedValueAbstractSet[Dimension], 

316 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

317 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

318 ): 

319 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

320 required.add(self) 

321 self._required = required.freeze() 

322 self._uniqueKeys = uniqueKeys 

323 

324 @property 

325 def required(self) -> NamedValueAbstractSet[Dimension]: 

326 # Docstring inherited from DimensionElement. 

327 return self._required 

328 

329 @property 

330 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

331 # Docstring inherited from Dimension. 

332 return self._uniqueKeys 

333 

334 

335class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

336 """A combination class that maps directly to a database table or query. 

337 

338 Parameters 

339 ---------- 

340 name : `str` 

341 Name of the dimension. 

342 storage : `dict` 

343 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

344 that will back this element in the registry (in a "cls" key) along 

345 with any other construction keyword arguments (in other keys). 

346 required : `NamedValueAbstractSet` [ `Dimension` ] 

347 Dimensions whose keys define the compound primary key for this 

348 combinations's (logical) table, as well as references to their own 

349 tables. 

350 implied : `NamedValueAbstractSet` [ `Dimension` ] 

351 Dimensions whose keys are included in this combinations's (logical) 

352 table as foreign keys. 

353 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

354 Field specifications for all non-key fields in this combination's 

355 table. 

356 alwaysJoin : `bool`, optional 

357 If `True`, always include this element in any query or data ID in 

358 which its ``required`` dimensions appear, because it defines a 

359 relationship between those dimensions that must always be satisfied. 

360 

361 Notes 

362 ----- 

363 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

364 but it is the responsibility of 

365 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

366 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

367 

368 This class has a lot in common with `DatabaseDimension`, but they are 

369 expected to diverge in future changes, and the only way to make them share 

370 method implementations would be via multiple inheritance. Given the 

371 trivial nature of all of those implementations, this does not seem worth 

372 the drawbacks (particularly the constraints it imposes on constructor 

373 signatures). 

374 """ 

375 

376 def __init__( 

377 self, 

378 name: str, 

379 storage: dict, *, 

380 required: NamedValueAbstractSet[Dimension], 

381 implied: NamedValueAbstractSet[Dimension], 

382 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

383 alwaysJoin: bool, 

384 ): 

385 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

386 self._required = required 

387 self._alwaysJoin = alwaysJoin 

388 

389 @property 

390 def required(self) -> NamedValueAbstractSet[Dimension]: 

391 # Docstring inherited from DimensionElement. 

392 return self._required 

393 

394 @property 

395 def alwaysJoin(self) -> bool: 

396 # Docstring inherited from DimensionElement. 

397 return self._alwaysJoin 

398 

399 

400class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

401 """Construction visitor for database dimension and dimension combination. 

402 

403 Specifically, a construction visitor for `DatabaseDimension` and 

404 `DatabaseDimensionCombination`. 

405 

406 Parameters 

407 ---------- 

408 name : `str` 

409 Name of the dimension. 

410 storage : `dict` 

411 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

412 that will back this element in the registry (in a "cls" key) along 

413 with any other construction keyword arguments (in other keys). 

414 required : `Set` [ `Dimension` ] 

415 Names of dimensions whose keys define the compound primary key for this 

416 element's (logical) table, as well as references to their own 

417 tables. 

418 implied : `Set` [ `Dimension` ] 

419 Names of dimension whose keys are included in this elements's 

420 (logical) table as foreign keys. 

421 metadata : `Iterable` [ `ddl.FieldSpec` ] 

422 Field specifications for all non-key fields in this element's table. 

423 uniqueKeys : `Iterable` [ `ddl.FieldSpec` ] 

424 Fields that can each be used to uniquely identify this dimension (given 

425 values for all required dimensions). The first of these is used as 

426 (part of) this dimension's table's primary key, while others are used 

427 to define unique constraints. Should be empty for 

428 `DatabaseDimensionCombination` definitions. 

429 alwaysJoin : `bool`, optional 

430 If `True`, always include this element in any query or data ID in 

431 which its ``required`` dimensions appear, because it defines a 

432 relationship between those dimensions that must always be satisfied. 

433 Should only be provided when a `DimensionCombination` is being 

434 constructed. 

435 """ 

436 

437 def __init__( 

438 self, 

439 name: str, 

440 storage: dict, 

441 required: Set[str], 

442 implied: Set[str], 

443 metadata: Iterable[ddl.FieldSpec] = (), 

444 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

445 alwaysJoin: bool = False, 

446 ): 

447 super().__init__(name) 

448 self._storage = storage 

449 self._required = required 

450 self._implied = implied 

451 self._metadata = NamedValueSet(metadata).freeze() 

452 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

453 self._alwaysJoin = alwaysJoin 

454 

455 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

456 # Docstring inherited from DimensionConstructionVisitor. 

457 return not ( 

458 self._required.isdisjoint(others) 

459 and self._implied.isdisjoint(others) 

460 ) 

461 

462 def visit(self, builder: DimensionConstructionBuilder) -> None: 

463 # Docstring inherited from DimensionConstructionVisitor. 

464 # Expand required dependencies. 

465 for name in tuple(self._required): # iterate over copy 

466 self._required.update(builder.dimensions[name].required.names) 

467 # Transform required and implied Dimension names into instances, 

468 # and reorder to match builder's order. 

469 required: NamedValueSet[Dimension] = NamedValueSet() 

470 implied: NamedValueSet[Dimension] = NamedValueSet() 

471 for dimension in builder.dimensions: 

472 if dimension.name in self._required: 

473 required.add(dimension) 

474 if dimension.name in self._implied: 

475 implied.add(dimension) 

476 

477 if self._uniqueKeys: 

478 if self._alwaysJoin: 

479 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

480 # Special handling for creating Dimension instances. 

481 dimension = DatabaseDimension( 

482 self.name, 

483 storage=self._storage, 

484 required=required, 

485 implied=implied.freeze(), 

486 metadata=self._metadata, 

487 uniqueKeys=self._uniqueKeys, 

488 ) 

489 builder.dimensions.add(dimension) 

490 builder.elements.add(dimension) 

491 else: 

492 # Special handling for creating DimensionCombination instances. 

493 combination = DatabaseDimensionCombination( 

494 self.name, 

495 storage=self._storage, 

496 required=required, 

497 implied=implied.freeze(), 

498 metadata=self._metadata, 

499 alwaysJoin=self._alwaysJoin, 

500 ) 

501 builder.elements.add(combination)