Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from types import MappingProxyType 

32from typing import ( 

33 AbstractSet, 

34 Dict, 

35 Iterable, 

36 Mapping, 

37 Optional, 

38 Set, 

39 TYPE_CHECKING, 

40) 

41 

42from lsst.utils import doImport 

43 

44from .. import ddl 

45from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

46from ..utils import cached_getter 

47from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

48 

49from ._elements import Dimension, DimensionCombination, DimensionElement 

50from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

51 

52if TYPE_CHECKING: 52 ↛ 53line 52 didn't jump to line 53, because the condition on line 52 was never true

53 from ._governor import GovernorDimension 

54 from ...registry.interfaces import ( 

55 Database, 

56 DatabaseDimensionRecordStorage, 

57 GovernorDimensionRecordStorage, 

58 StaticTablesContext, 

59 ) 

60 

61 

62class DatabaseTopologicalFamily(TopologicalFamily): 

63 """A `TopologicalFamily` implementation for the `DatabaseDimension` and 

64 `DatabaseDimensionCombination` objects that have direct database 

65 representations. 

66 

67 Parameters 

68 ---------- 

69 name : `str` 

70 Name of the family. 

71 space : `TopologicalSpace` 

72 Space in which this family's regions live. 

73 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

74 The members of this family, ordered according to the priority used 

75 in `choose` (first-choice member first). 

76 """ 

77 def __init__( 

78 self, 

79 name: str, 

80 space: TopologicalSpace, *, 

81 members: NamedValueAbstractSet[DimensionElement], 

82 ): 

83 super().__init__(name, space) 

84 self.members = members 

85 

86 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

87 # Docstring inherited from TopologicalFamily. 

88 for member in self.members: 

89 if member in endpoints: 

90 return member 

91 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

92 

93 @property # type: ignore 

94 @cached_getter 

95 def governor(self) -> GovernorDimension: 

96 """The `GovernorDimension` common to all members of this family 

97 (`GovernorDimension`). 

98 """ 

99 governors = set(m.governor for m in self.members) 

100 if None in governors: 

101 raise RuntimeError( 

102 f"Bad {self.space.name} family definition {self.name}: at least one member " 

103 f"in {self.members} has no GovernorDimension dependency." 

104 ) 

105 try: 

106 (result,) = governors 

107 except ValueError: 

108 raise RuntimeError( 

109 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

110 f"in {self.members}." 

111 ) from None 

112 return result # type: ignore 

113 

114 members: NamedValueAbstractSet[DimensionElement] 

115 """The members of this family, ordered according to the priority used in 

116 `choose` (first-choice member first). 

117 """ 

118 

119 

120class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

121 """A construction visitor for `DatabaseTopologicalFamily`. 

122 

123 This visitor depends on (and is thus visited after) its members. 

124 

125 Parameters 

126 ---------- 

127 name : `str` 

128 Name of the family. 

129 members : `Iterable` [ `str` ] 

130 The names of the members of this family, ordered according to the 

131 priority used in `choose` (first-choice member first). 

132 """ 

133 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

134 super().__init__(name) 

135 self._space = space 

136 self._members = tuple(members) 

137 

138 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

139 # Docstring inherited from DimensionConstructionVisitor. 

140 return not others.isdisjoint(self._members) 

141 

142 def visit(self, builder: DimensionConstructionBuilder) -> None: 

143 # Docstring inherited from DimensionConstructionVisitor. 

144 members = NamedValueSet(builder.elements[name] for name in self._members) 

145 family = DatabaseTopologicalFamily( 

146 self.name, 

147 self._space, 

148 members=members.freeze() 

149 ) 

150 builder.topology[self._space].add(family) 

151 for member in members: 

152 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination)) 

153 other = member._topology.setdefault(self._space, family) 

154 if other is not family: 

155 raise RuntimeError(f"{member.name} is declared to be a member of (at least) two " 

156 f"{self._space.name} families: {other.name} and {family.name}.") 

157 

158 

159class DatabaseDimensionElement(DimensionElement): 

160 """An intermediate base class for `DimensionElement` classes whose 

161 instances that map directly to a database table or query. 

162 

163 Parameters 

164 ---------- 

165 name : `str` 

166 Name of the dimension. 

167 storage : `dict` 

168 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

169 that will back this element in the registry (in a "cls" key) along 

170 with any other construction keyword arguments (in other keys). 

171 implied : `NamedValueAbstractSet` [ `Dimension` ] 

172 Other dimensions whose keys are included in this dimension's (logical) 

173 table as foreign keys. 

174 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

175 Field specifications for all non-key fields in this dimension's table. 

176 """ 

177 

178 def __init__( 

179 self, 

180 name: str, 

181 storage: dict, *, 

182 implied: NamedValueAbstractSet[Dimension], 

183 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

184 ): 

185 self._name = name 

186 self._storage = storage 

187 self._implied = implied 

188 self._metadata = metadata 

189 self._topology: Dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

190 

191 @property 

192 def name(self) -> str: 

193 # Docstring inherited from TopoogicalRelationshipEndpoint. 

194 return self._name 

195 

196 @property 

197 def implied(self) -> NamedValueAbstractSet[Dimension]: 

198 # Docstring inherited from DimensionElement. 

199 return self._implied 

200 

201 @property 

202 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

203 # Docstring inherited from DimensionElement. 

204 return self._metadata 

205 

206 @property 

207 def viewOf(self) -> Optional[str]: 

208 # Docstring inherited from DimensionElement. 

209 # This is a bit encapsulation-breaking; these storage config values 

210 # are supposed to be opaque here, and just forwarded on to some 

211 # DimensionRecordStorage implementation. The long-term fix is to 

212 # move viewOf entirely, by changing the code that relies on it to rely 

213 # on the DimensionRecordStorage object instead. 

214 storage = self._storage.get("nested") 

215 if storage is None: 

216 storage = self._storage 

217 return storage.get("view_of") 

218 

219 @property 

220 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

221 # Docstring inherited from TopologicalRelationshipEndpoint 

222 return MappingProxyType(self._topology) 

223 

224 @property 

225 def spatial(self) -> Optional[DatabaseTopologicalFamily]: 

226 # Docstring inherited from TopologicalRelationshipEndpoint 

227 return self.topology.get(TopologicalSpace.SPATIAL) 

228 

229 @property 

230 def temporal(self) -> Optional[DatabaseTopologicalFamily]: 

231 # Docstring inherited from TopologicalRelationshipEndpoint 

232 return self.topology.get(TopologicalSpace.TEMPORAL) 

233 

234 def makeStorage( 

235 self, 

236 db: Database, *, 

237 context: Optional[StaticTablesContext] = None, 

238 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

239 ) -> DatabaseDimensionRecordStorage: 

240 """Construct the `DimensionRecordStorage` instance that should 

241 be used to back this element in a registry. 

242 

243 Parameters 

244 ---------- 

245 db : `Database` 

246 Interface to the underlying database engine and namespace. 

247 context : `StaticTablesContext`, optional 

248 If provided, an object to use to create any new tables. If not 

249 provided, ``db.ensureTableExists`` should be used instead. 

250 governors : `NamedKeyMapping` 

251 Mapping from `GovernorDimension` to the record storage backend for 

252 that dimension, containing all governor dimensions. 

253 

254 Returns 

255 ------- 

256 storage : `DatabaseDimensionRecordStorage` 

257 Storage object that should back this element in a registry. 

258 """ 

259 from ...registry.interfaces import DatabaseDimensionRecordStorage 

260 cls = doImport(self._storage["cls"]) 

261 assert issubclass(cls, DatabaseDimensionRecordStorage) 

262 return cls.initialize(db, self, context=context, config=self._storage, governors=governors) 

263 

264 

265class DatabaseDimension(Dimension, DatabaseDimensionElement): 

266 """A `Dimension` implementation that maps directly to a database table or 

267 query. 

268 

269 Parameters 

270 ---------- 

271 name : `str` 

272 Name of the dimension. 

273 storage : `dict` 

274 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

275 that will back this element in the registry (in a "cls" key) along 

276 with any other construction keyword arguments (in other keys). 

277 required : `NamedValueSet` [ `Dimension` ] 

278 Other dimensions whose keys are part of the compound primary key for 

279 this dimension's (logical) table, as well as references to their own 

280 tables. The ``required`` parameter does not include ``self`` (it 

281 can't, of course), but the corresponding attribute does - it is added 

282 by the constructor. 

283 implied : `NamedValueAbstractSet` [ `Dimension` ] 

284 Other dimensions whose keys are included in this dimension's (logical) 

285 table as foreign keys. 

286 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

287 Field specifications for all non-key fields in this dimension's table. 

288 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

289 Fields that can each be used to uniquely identify this dimension (given 

290 values for all required dimensions). The first of these is used as 

291 (part of) this dimension's table's primary key, while others are used 

292 to define unique constraints. 

293 

294 Notes 

295 ----- 

296 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

297 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

298 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

299 members. 

300 """ 

301 def __init__( 

302 self, 

303 name: str, 

304 storage: dict, *, 

305 required: NamedValueSet[Dimension], 

306 implied: NamedValueAbstractSet[Dimension], 

307 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

308 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

309 ): 

310 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

311 required.add(self) 

312 self._required = required.freeze() 

313 self._uniqueKeys = uniqueKeys 

314 

315 @property 

316 def required(self) -> NamedValueAbstractSet[Dimension]: 

317 # Docstring inherited from DimensionElement. 

318 return self._required 

319 

320 @property 

321 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

322 # Docstring inherited from Dimension. 

323 return self._uniqueKeys 

324 

325 

326class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

327 """A `DimensionCombination` implementation that maps directly to a database 

328 table or query. 

329 

330 Parameters 

331 ---------- 

332 name : `str` 

333 Name of the dimension. 

334 storage : `dict` 

335 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

336 that will back this element in the registry (in a "cls" key) along 

337 with any other construction keyword arguments (in other keys). 

338 required : `NamedValueAbstractSet` [ `Dimension` ] 

339 Dimensions whose keys define the compound primary key for this 

340 combinations's (logical) table, as well as references to their own 

341 tables. 

342 implied : `NamedValueAbstractSet` [ `Dimension` ] 

343 Dimensions whose keys are included in this combinations's (logical) 

344 table as foreign keys. 

345 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

346 Field specifications for all non-key fields in this combination's 

347 table. 

348 alwaysJoin : `bool`, optional 

349 If `True`, always include this element in any query or data ID in 

350 which its ``required`` dimensions appear, because it defines a 

351 relationship between those dimensions that must always be satisfied. 

352 

353 Notes 

354 ----- 

355 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

356 but it is the responsibility of 

357 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

358 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

359 

360 This class has a lot in common with `DatabaseDimension`, but they are 

361 expected to diverge in future changes, and the only way to make them share 

362 method implementations would be via multiple inheritance. Given the 

363 trivial nature of all of those implementations, this does not seem worth 

364 the drawbacks (particularly the constraints it imposes on constructor 

365 signatures). 

366 """ 

367 def __init__( 

368 self, 

369 name: str, 

370 storage: dict, *, 

371 required: NamedValueAbstractSet[Dimension], 

372 implied: NamedValueAbstractSet[Dimension], 

373 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

374 alwaysJoin: bool, 

375 ): 

376 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

377 self._required = required 

378 self._alwaysJoin = alwaysJoin 

379 

380 @property 

381 def required(self) -> NamedValueAbstractSet[Dimension]: 

382 # Docstring inherited from DimensionElement. 

383 return self._required 

384 

385 @property 

386 def alwaysJoin(self) -> bool: 

387 # Docstring inherited from DimensionElement. 

388 return self._alwaysJoin 

389 

390 

391class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

392 """A construction visitor for `DatabaseDimension` and 

393 `DatabaseDimensionCombination`. 

394 

395 Parameters 

396 ---------- 

397 name : `str` 

398 Name of the dimension. 

399 storage : `dict` 

400 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

401 that will back this element in the registry (in a "cls" key) along 

402 with any other construction keyword arguments (in other keys). 

403 required : `Set` [ `Dimension` ] 

404 Names of dimensions whose keys define the compound primary key for this 

405 element's (logical) table, as well as references to their own 

406 tables. 

407 implied : `Set` [ `Dimension` ] 

408 Names of dimension whose keys are included in this elements's 

409 (logical) table as foreign keys. 

410 metadata : `Iterable` [ `ddl.FieldSpec` ] 

411 Field specifications for all non-key fields in this element's table. 

412 uniqueKeys : `Iterable` [ `ddl.FieldSpec` ] 

413 Fields that can each be used to uniquely identify this dimension (given 

414 values for all required dimensions). The first of these is used as 

415 (part of) this dimension's table's primary key, while others are used 

416 to define unique constraints. Should be empty for 

417 `DatabaseDimensionCombination` definitions. 

418 alwaysJoin : `bool`, optional 

419 If `True`, always include this element in any query or data ID in 

420 which its ``required`` dimensions appear, because it defines a 

421 relationship between those dimensions that must always be satisfied. 

422 Should only be provided when a `DimensionCombination` is being 

423 constructed. 

424 """ 

425 def __init__( 

426 self, 

427 name: str, 

428 storage: dict, 

429 required: Set[str], 

430 implied: Set[str], 

431 metadata: Iterable[ddl.FieldSpec] = (), 

432 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

433 alwaysJoin: bool = False, 

434 ): 

435 super().__init__(name) 

436 self._storage = storage 

437 self._required = required 

438 self._implied = implied 

439 self._metadata = NamedValueSet(metadata).freeze() 

440 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

441 self._alwaysJoin = alwaysJoin 

442 

443 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

444 # Docstring inherited from DimensionConstructionVisitor. 

445 return not ( 

446 self._required.isdisjoint(others) 

447 and self._implied.isdisjoint(others) 

448 ) 

449 

450 def visit(self, builder: DimensionConstructionBuilder) -> None: 

451 # Docstring inherited from DimensionConstructionVisitor. 

452 # Expand required dependencies. 

453 for name in tuple(self._required): # iterate over copy 

454 self._required.update(builder.dimensions[name].required.names) 

455 # Transform required and implied Dimension names into instances, 

456 # and reorder to match builder's order. 

457 required: NamedValueSet[Dimension] = NamedValueSet() 

458 implied: NamedValueSet[Dimension] = NamedValueSet() 

459 for dimension in builder.dimensions: 

460 if dimension.name in self._required: 

461 required.add(dimension) 

462 if dimension.name in self._implied: 

463 implied.add(dimension) 

464 

465 if self._uniqueKeys: 

466 if self._alwaysJoin: 

467 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

468 # Special handling for creating Dimension instances. 

469 dimension = DatabaseDimension( 

470 self.name, 

471 storage=self._storage, 

472 required=required, 

473 implied=implied.freeze(), 

474 metadata=self._metadata, 

475 uniqueKeys=self._uniqueKeys, 

476 ) 

477 builder.dimensions.add(dimension) 

478 builder.elements.add(dimension) 

479 else: 

480 # Special handling for creating DimensionCombination instances. 

481 combination = DatabaseDimensionCombination( 

482 self.name, 

483 storage=self._storage, 

484 required=required, 

485 implied=implied.freeze(), 

486 metadata=self._metadata, 

487 alwaysJoin=self._alwaysJoin, 

488 ) 

489 builder.elements.add(combination)