Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 34%

140 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-17 02:01 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from types import MappingProxyType 

32from typing import TYPE_CHECKING, AbstractSet, Dict, Iterable, Mapping, Optional, Set 

33 

34from lsst.utils import doImportType 

35from lsst.utils.classes import cached_getter 

36 

37from .. import ddl 

38from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

39from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

40from ._elements import Dimension, DimensionCombination, DimensionElement 

41from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from ...registry.interfaces import ( 

45 Database, 

46 DatabaseDimensionRecordStorage, 

47 GovernorDimensionRecordStorage, 

48 StaticTablesContext, 

49 ) 

50 from ._governor import GovernorDimension 

51 

52 

53class DatabaseTopologicalFamily(TopologicalFamily): 

54 """Database topological family implementation. 

55 

56 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

57 `DatabaseDimensionCombination` objects that have direct database 

58 representations. 

59 

60 Parameters 

61 ---------- 

62 name : `str` 

63 Name of the family. 

64 space : `TopologicalSpace` 

65 Space in which this family's regions live. 

66 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

67 The members of this family, ordered according to the priority used 

68 in `choose` (first-choice member first). 

69 """ 

70 

71 def __init__( 

72 self, 

73 name: str, 

74 space: TopologicalSpace, 

75 *, 

76 members: NamedValueAbstractSet[DimensionElement], 

77 ): 

78 super().__init__(name, space) 

79 self.members = members 

80 

81 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

82 # Docstring inherited from TopologicalFamily. 

83 for member in self.members: 

84 if member in endpoints: 

85 return member 

86 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

87 

88 @property 

89 @cached_getter 

90 def governor(self) -> GovernorDimension: 

91 """Return `GovernorDimension` common to all members of this family. 

92 

93 (`GovernorDimension`). 

94 """ 

95 governors = set(m.governor for m in self.members) 

96 if None in governors: 

97 raise RuntimeError( 

98 f"Bad {self.space.name} family definition {self.name}: at least one member " 

99 f"in {self.members} has no GovernorDimension dependency." 

100 ) 

101 try: 

102 (result,) = governors 

103 except ValueError: 

104 raise RuntimeError( 

105 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

106 f"in {self.members}." 

107 ) from None 

108 return result # type: ignore 

109 

110 members: NamedValueAbstractSet[DimensionElement] 

111 """The members of this family, ordered according to the priority used in 

112 `choose` (first-choice member first). 

113 """ 

114 

115 

116class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

117 """A construction visitor for `DatabaseTopologicalFamily`. 

118 

119 This visitor depends on (and is thus visited after) its members. 

120 

121 Parameters 

122 ---------- 

123 name : `str` 

124 Name of the family. 

125 members : `Iterable` [ `str` ] 

126 The names of the members of this family, ordered according to the 

127 priority used in `choose` (first-choice member first). 

128 """ 

129 

130 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

131 super().__init__(name) 

132 self._space = space 

133 self._members = tuple(members) 

134 

135 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

136 # Docstring inherited from DimensionConstructionVisitor. 

137 return not others.isdisjoint(self._members) 

138 

139 def visit(self, builder: DimensionConstructionBuilder) -> None: 

140 # Docstring inherited from DimensionConstructionVisitor. 

141 members = NamedValueSet(builder.elements[name] for name in self._members) 

142 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

143 builder.topology[self._space].add(family) 

144 for member in members: 

145 assert isinstance(member, (DatabaseDimension, DatabaseDimensionCombination)) 

146 other = member._topology.setdefault(self._space, family) 

147 if other is not family: 

148 raise RuntimeError( 

149 f"{member.name} is declared to be a member of (at least) two " 

150 f"{self._space.name} families: {other.name} and {family.name}." 

151 ) 

152 

153 

154class DatabaseDimensionElement(DimensionElement): 

155 """An intermediate base class for `DimensionElement` database classes. 

156 

157 Theese classes are ones whose instances map directly to a database 

158 table or query. 

159 

160 Parameters 

161 ---------- 

162 name : `str` 

163 Name of the dimension. 

164 storage : `dict` 

165 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

166 that will back this element in the registry (in a "cls" key) along 

167 with any other construction keyword arguments (in other keys). 

168 implied : `NamedValueAbstractSet` [ `Dimension` ] 

169 Other dimensions whose keys are included in this dimension's (logical) 

170 table as foreign keys. 

171 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

172 Field specifications for all non-key fields in this dimension's table. 

173 """ 

174 

175 def __init__( 

176 self, 

177 name: str, 

178 storage: dict, 

179 *, 

180 implied: NamedValueAbstractSet[Dimension], 

181 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

182 ): 

183 self._name = name 

184 self._storage = storage 

185 self._implied = implied 

186 self._metadata = metadata 

187 self._topology: Dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

188 

189 @property 

190 def name(self) -> str: 

191 # Docstring inherited from TopologicalRelationshipEndpoint. 

192 return self._name 

193 

194 @property 

195 def implied(self) -> NamedValueAbstractSet[Dimension]: 

196 # Docstring inherited from DimensionElement. 

197 return self._implied 

198 

199 @property 

200 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

201 # Docstring inherited from DimensionElement. 

202 return self._metadata 

203 

204 @property 

205 def viewOf(self) -> Optional[str]: 

206 # Docstring inherited from DimensionElement. 

207 # This is a bit encapsulation-breaking; these storage config values 

208 # are supposed to be opaque here, and just forwarded on to some 

209 # DimensionRecordStorage implementation. The long-term fix is to 

210 # move viewOf entirely, by changing the code that relies on it to rely 

211 # on the DimensionRecordStorage object instead. 

212 storage = self._storage.get("nested") 

213 if storage is None: 

214 storage = self._storage 

215 return storage.get("view_of") 

216 

217 @property 

218 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

219 # Docstring inherited from TopologicalRelationshipEndpoint 

220 return MappingProxyType(self._topology) 

221 

222 @property 

223 def spatial(self) -> Optional[DatabaseTopologicalFamily]: 

224 # Docstring inherited from TopologicalRelationshipEndpoint 

225 return self.topology.get(TopologicalSpace.SPATIAL) 

226 

227 @property 

228 def temporal(self) -> Optional[DatabaseTopologicalFamily]: 

229 # Docstring inherited from TopologicalRelationshipEndpoint 

230 return self.topology.get(TopologicalSpace.TEMPORAL) 

231 

232 def makeStorage( 

233 self, 

234 db: Database, 

235 *, 

236 context: Optional[StaticTablesContext] = None, 

237 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

238 ) -> DatabaseDimensionRecordStorage: 

239 """Make the dimension record storage instance for this database. 

240 

241 Constructs the `DimensionRecordStorage` instance that should 

242 be used to back this element in a registry. 

243 

244 Parameters 

245 ---------- 

246 db : `Database` 

247 Interface to the underlying database engine and namespace. 

248 context : `StaticTablesContext`, optional 

249 If provided, an object to use to create any new tables. If not 

250 provided, ``db.ensureTableExists`` should be used instead. 

251 governors : `NamedKeyMapping` 

252 Mapping from `GovernorDimension` to the record storage backend for 

253 that dimension, containing all governor dimensions. 

254 

255 Returns 

256 ------- 

257 storage : `DatabaseDimensionRecordStorage` 

258 Storage object that should back this element in a registry. 

259 """ 

260 from ...registry.interfaces import DatabaseDimensionRecordStorage 

261 

262 cls = doImportType(self._storage["cls"]) 

263 assert issubclass(cls, DatabaseDimensionRecordStorage) 

264 return cls.initialize(db, self, context=context, config=self._storage, governors=governors) 

265 

266 

267class DatabaseDimension(Dimension, DatabaseDimensionElement): 

268 """A `Dimension` class that maps directly to a database table or query. 

269 

270 Parameters 

271 ---------- 

272 name : `str` 

273 Name of the dimension. 

274 storage : `dict` 

275 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

276 that will back this element in the registry (in a "cls" key) along 

277 with any other construction keyword arguments (in other keys). 

278 required : `NamedValueSet` [ `Dimension` ] 

279 Other dimensions whose keys are part of the compound primary key for 

280 this dimension's (logical) table, as well as references to their own 

281 tables. The ``required`` parameter does not include ``self`` (it 

282 can't, of course), but the corresponding attribute does - it is added 

283 by the constructor. 

284 implied : `NamedValueAbstractSet` [ `Dimension` ] 

285 Other dimensions whose keys are included in this dimension's (logical) 

286 table as foreign keys. 

287 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

288 Field specifications for all non-key fields in this dimension's table. 

289 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

290 Fields that can each be used to uniquely identify this dimension (given 

291 values for all required dimensions). The first of these is used as 

292 (part of) this dimension's table's primary key, while others are used 

293 to define unique constraints. 

294 

295 Notes 

296 ----- 

297 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

298 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

299 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

300 members. 

301 """ 

302 

303 def __init__( 

304 self, 

305 name: str, 

306 storage: dict, 

307 *, 

308 required: NamedValueSet[Dimension], 

309 implied: NamedValueAbstractSet[Dimension], 

310 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

311 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

312 ): 

313 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

314 required.add(self) 

315 self._required = required.freeze() 

316 self._uniqueKeys = uniqueKeys 

317 

318 @property 

319 def required(self) -> NamedValueAbstractSet[Dimension]: 

320 # Docstring inherited from DimensionElement. 

321 return self._required 

322 

323 @property 

324 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

325 # Docstring inherited from Dimension. 

326 return self._uniqueKeys 

327 

328 

329class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

330 """A combination class that maps directly to a database table or query. 

331 

332 Parameters 

333 ---------- 

334 name : `str` 

335 Name of the dimension. 

336 storage : `dict` 

337 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

338 that will back this element in the registry (in a "cls" key) along 

339 with any other construction keyword arguments (in other keys). 

340 required : `NamedValueAbstractSet` [ `Dimension` ] 

341 Dimensions whose keys define the compound primary key for this 

342 combinations's (logical) table, as well as references to their own 

343 tables. 

344 implied : `NamedValueAbstractSet` [ `Dimension` ] 

345 Dimensions whose keys are included in this combinations's (logical) 

346 table as foreign keys. 

347 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

348 Field specifications for all non-key fields in this combination's 

349 table. 

350 alwaysJoin : `bool`, optional 

351 If `True`, always include this element in any query or data ID in 

352 which its ``required`` dimensions appear, because it defines a 

353 relationship between those dimensions that must always be satisfied. 

354 

355 Notes 

356 ----- 

357 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

358 but it is the responsibility of 

359 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

360 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

361 

362 This class has a lot in common with `DatabaseDimension`, but they are 

363 expected to diverge in future changes, and the only way to make them share 

364 method implementations would be via multiple inheritance. Given the 

365 trivial nature of all of those implementations, this does not seem worth 

366 the drawbacks (particularly the constraints it imposes on constructor 

367 signatures). 

368 """ 

369 

370 def __init__( 

371 self, 

372 name: str, 

373 storage: dict, 

374 *, 

375 required: NamedValueAbstractSet[Dimension], 

376 implied: NamedValueAbstractSet[Dimension], 

377 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

378 alwaysJoin: bool, 

379 ): 

380 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

381 self._required = required 

382 self._alwaysJoin = alwaysJoin 

383 

384 @property 

385 def required(self) -> NamedValueAbstractSet[Dimension]: 

386 # Docstring inherited from DimensionElement. 

387 return self._required 

388 

389 @property 

390 def alwaysJoin(self) -> bool: 

391 # Docstring inherited from DimensionElement. 

392 return self._alwaysJoin 

393 

394 

395class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

396 """Construction visitor for database dimension and dimension combination. 

397 

398 Specifically, a construction visitor for `DatabaseDimension` and 

399 `DatabaseDimensionCombination`. 

400 

401 Parameters 

402 ---------- 

403 name : `str` 

404 Name of the dimension. 

405 storage : `dict` 

406 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

407 that will back this element in the registry (in a "cls" key) along 

408 with any other construction keyword arguments (in other keys). 

409 required : `Set` [ `Dimension` ] 

410 Names of dimensions whose keys define the compound primary key for this 

411 element's (logical) table, as well as references to their own 

412 tables. 

413 implied : `Set` [ `Dimension` ] 

414 Names of dimension whose keys are included in this elements's 

415 (logical) table as foreign keys. 

416 metadata : `Iterable` [ `ddl.FieldSpec` ] 

417 Field specifications for all non-key fields in this element's table. 

418 uniqueKeys : `Iterable` [ `ddl.FieldSpec` ] 

419 Fields that can each be used to uniquely identify this dimension (given 

420 values for all required dimensions). The first of these is used as 

421 (part of) this dimension's table's primary key, while others are used 

422 to define unique constraints. Should be empty for 

423 `DatabaseDimensionCombination` definitions. 

424 alwaysJoin : `bool`, optional 

425 If `True`, always include this element in any query or data ID in 

426 which its ``required`` dimensions appear, because it defines a 

427 relationship between those dimensions that must always be satisfied. 

428 Should only be provided when a `DimensionCombination` is being 

429 constructed. 

430 """ 

431 

432 def __init__( 

433 self, 

434 name: str, 

435 storage: dict, 

436 required: Set[str], 

437 implied: Set[str], 

438 metadata: Iterable[ddl.FieldSpec] = (), 

439 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

440 alwaysJoin: bool = False, 

441 ): 

442 super().__init__(name) 

443 self._storage = storage 

444 self._required = required 

445 self._implied = implied 

446 self._metadata = NamedValueSet(metadata).freeze() 

447 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

448 self._alwaysJoin = alwaysJoin 

449 

450 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

451 # Docstring inherited from DimensionConstructionVisitor. 

452 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

453 

454 def visit(self, builder: DimensionConstructionBuilder) -> None: 

455 # Docstring inherited from DimensionConstructionVisitor. 

456 # Expand required dependencies. 

457 for name in tuple(self._required): # iterate over copy 

458 self._required.update(builder.dimensions[name].required.names) 

459 # Transform required and implied Dimension names into instances, 

460 # and reorder to match builder's order. 

461 required: NamedValueSet[Dimension] = NamedValueSet() 

462 implied: NamedValueSet[Dimension] = NamedValueSet() 

463 for dimension in builder.dimensions: 

464 if dimension.name in self._required: 

465 required.add(dimension) 

466 if dimension.name in self._implied: 

467 implied.add(dimension) 

468 

469 if self._uniqueKeys: 

470 if self._alwaysJoin: 

471 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

472 # Special handling for creating Dimension instances. 

473 dimension = DatabaseDimension( 

474 self.name, 

475 storage=self._storage, 

476 required=required, 

477 implied=implied.freeze(), 

478 metadata=self._metadata, 

479 uniqueKeys=self._uniqueKeys, 

480 ) 

481 builder.dimensions.add(dimension) 

482 builder.elements.add(dimension) 

483 else: 

484 # Special handling for creating DimensionCombination instances. 

485 combination = DatabaseDimensionCombination( 

486 self.name, 

487 storage=self._storage, 

488 required=required, 

489 implied=implied.freeze(), 

490 metadata=self._metadata, 

491 alwaysJoin=self._alwaysJoin, 

492 ) 

493 builder.elements.add(combination)