Coverage for python/lsst/daf/butler/core/dimensions/_database.py: 44%

143 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "DatabaseDimension", 

26 "DatabaseDimensionCombination", 

27 "DatabaseDimensionElement", 

28 "DatabaseTopologicalFamily", 

29) 

30 

31from collections.abc import Iterable, Mapping, Set 

32from types import MappingProxyType 

33from typing import TYPE_CHECKING 

34 

35from lsst.utils import doImportType 

36from lsst.utils.classes import cached_getter 

37 

38from .. import ddl 

39from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace 

40from ..named import NamedKeyMapping, NamedValueAbstractSet, NamedValueSet 

41from ._elements import Dimension, DimensionCombination, DimensionElement 

42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

43 

44if TYPE_CHECKING: 

45 from ...registry.interfaces import ( 

46 Database, 

47 DatabaseDimensionRecordStorage, 

48 GovernorDimensionRecordStorage, 

49 StaticTablesContext, 

50 ) 

51 from ._governor import GovernorDimension 

52 

53 

54class DatabaseTopologicalFamily(TopologicalFamily): 

55 """Database topological family implementation. 

56 

57 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

58 `DatabaseDimensionCombination` objects that have direct database 

59 representations. 

60 

61 Parameters 

62 ---------- 

63 name : `str` 

64 Name of the family. 

65 space : `TopologicalSpace` 

66 Space in which this family's regions live. 

67 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

68 The members of this family, ordered according to the priority used 

69 in `choose` (first-choice member first). 

70 """ 

71 

72 def __init__( 

73 self, 

74 name: str, 

75 space: TopologicalSpace, 

76 *, 

77 members: NamedValueAbstractSet[DimensionElement], 

78 ): 

79 super().__init__(name, space) 

80 self.members = members 

81 

82 def choose(self, endpoints: NamedValueAbstractSet[TopologicalRelationshipEndpoint]) -> DimensionElement: 

83 # Docstring inherited from TopologicalFamily. 

84 for member in self.members: 

85 if member in endpoints: 

86 return member 

87 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

88 

89 @property 

90 @cached_getter 

91 def governor(self) -> GovernorDimension: 

92 """Return `GovernorDimension` common to all members of this family. 

93 

94 (`GovernorDimension`). 

95 """ 

96 governors = {m.governor for m in self.members} 

97 if None in governors: 

98 raise RuntimeError( 

99 f"Bad {self.space.name} family definition {self.name}: at least one member " 

100 f"in {self.members} has no GovernorDimension dependency." 

101 ) 

102 try: 

103 (result,) = governors 

104 except ValueError: 

105 raise RuntimeError( 

106 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

107 f"in {self.members}." 

108 ) from None 

109 return result # type: ignore 

110 

111 members: NamedValueAbstractSet[DimensionElement] 

112 """The members of this family, ordered according to the priority used in 

113 `choose` (first-choice member first). 

114 """ 

115 

116 

117class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

118 """A construction visitor for `DatabaseTopologicalFamily`. 

119 

120 This visitor depends on (and is thus visited after) its members. 

121 

122 Parameters 

123 ---------- 

124 name : `str` 

125 Name of the family. 

126 members : `~collections.abc.Iterable` [ `str` ] 

127 The names of the members of this family, ordered according to the 

128 priority used in `choose` (first-choice member first). 

129 """ 

130 

131 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

132 super().__init__(name) 

133 self._space = space 

134 self._members = tuple(members) 

135 

136 def hasDependenciesIn(self, others: Set[str]) -> bool: 

137 # Docstring inherited from DimensionConstructionVisitor. 

138 return not others.isdisjoint(self._members) 

139 

140 def visit(self, builder: DimensionConstructionBuilder) -> None: 

141 # Docstring inherited from DimensionConstructionVisitor. 

142 members = NamedValueSet(builder.elements[name] for name in self._members) 

143 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

144 builder.topology[self._space].add(family) 

145 for member in members: 

146 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination) 

147 other = member._topology.setdefault(self._space, family) 

148 if other is not family: 

149 raise RuntimeError( 

150 f"{member.name} is declared to be a member of (at least) two " 

151 f"{self._space.name} families: {other.name} and {family.name}." 

152 ) 

153 

154 

155class DatabaseDimensionElement(DimensionElement): 

156 """An intermediate base class for `DimensionElement` database classes. 

157 

158 Theese classes are ones whose instances map directly to a database 

159 table or query. 

160 

161 Parameters 

162 ---------- 

163 name : `str` 

164 Name of the dimension. 

165 storage : `dict` 

166 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

167 that will back this element in the registry (in a "cls" key) along 

168 with any other construction keyword arguments (in other keys). 

169 implied : `NamedValueAbstractSet` [ `Dimension` ] 

170 Other dimensions whose keys are included in this dimension's (logical) 

171 table as foreign keys. 

172 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

173 Field specifications for all non-key fields in this dimension's table. 

174 """ 

175 

176 def __init__( 

177 self, 

178 name: str, 

179 storage: dict, 

180 *, 

181 implied: NamedValueAbstractSet[Dimension], 

182 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

183 ): 

184 self._name = name 

185 self._storage = storage 

186 self._implied = implied 

187 self._metadata = metadata 

188 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

189 

190 @property 

191 def name(self) -> str: 

192 # Docstring inherited from TopologicalRelationshipEndpoint. 

193 return self._name 

194 

195 @property 

196 def implied(self) -> NamedValueAbstractSet[Dimension]: 

197 # Docstring inherited from DimensionElement. 

198 return self._implied 

199 

200 @property 

201 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

202 # Docstring inherited from DimensionElement. 

203 return self._metadata 

204 

205 @property 

206 def viewOf(self) -> str | None: 

207 # Docstring inherited from DimensionElement. 

208 # This is a bit encapsulation-breaking; these storage config values 

209 # are supposed to be opaque here, and just forwarded on to some 

210 # DimensionRecordStorage implementation. The long-term fix is to 

211 # move viewOf entirely, by changing the code that relies on it to rely 

212 # on the DimensionRecordStorage object instead. 

213 storage = self._storage.get("nested") 

214 if storage is None: 

215 storage = self._storage 

216 return storage.get("view_of") 

217 

218 @property 

219 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

220 # Docstring inherited from TopologicalRelationshipEndpoint 

221 return MappingProxyType(self._topology) 

222 

223 @property 

224 def spatial(self) -> DatabaseTopologicalFamily | None: 

225 # Docstring inherited from TopologicalRelationshipEndpoint 

226 return self.topology.get(TopologicalSpace.SPATIAL) 

227 

228 @property 

229 def temporal(self) -> DatabaseTopologicalFamily | None: 

230 # Docstring inherited from TopologicalRelationshipEndpoint 

231 return self.topology.get(TopologicalSpace.TEMPORAL) 

232 

233 def makeStorage( 

234 self, 

235 db: Database, 

236 *, 

237 context: StaticTablesContext | None = None, 

238 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

239 view_target: DatabaseDimensionRecordStorage | None = None, 

240 ) -> DatabaseDimensionRecordStorage: 

241 """Make the dimension record storage instance for this database. 

242 

243 Constructs the `DimensionRecordStorage` instance that should 

244 be used to back this element in a registry. 

245 

246 Parameters 

247 ---------- 

248 db : `Database` 

249 Interface to the underlying database engine and namespace. 

250 context : `StaticTablesContext`, optional 

251 If provided, an object to use to create any new tables. If not 

252 provided, ``db.ensureTableExists`` should be used instead. 

253 governors : `NamedKeyMapping` 

254 Mapping from `GovernorDimension` to the record storage backend for 

255 that dimension, containing all governor dimensions. 

256 view_target : `DatabaseDimensionRecordStorage`, optional 

257 Storage object for the element this target's storage is a view of 

258 (i.e. when `viewOf` is not `None`). 

259 

260 Returns 

261 ------- 

262 storage : `DatabaseDimensionRecordStorage` 

263 Storage object that should back this element in a registry. 

264 """ 

265 from ...registry.interfaces import DatabaseDimensionRecordStorage 

266 

267 cls = doImportType(self._storage["cls"]) 

268 assert issubclass(cls, DatabaseDimensionRecordStorage) 

269 return cls.initialize( 

270 db, 

271 self, 

272 context=context, 

273 config=self._storage, 

274 governors=governors, 

275 view_target=view_target, 

276 ) 

277 

278 

279class DatabaseDimension(Dimension, DatabaseDimensionElement): 

280 """A `Dimension` class that maps directly to a database table or query. 

281 

282 Parameters 

283 ---------- 

284 name : `str` 

285 Name of the dimension. 

286 storage : `dict` 

287 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

288 that will back this element in the registry (in a "cls" key) along 

289 with any other construction keyword arguments (in other keys). 

290 required : `NamedValueSet` [ `Dimension` ] 

291 Other dimensions whose keys are part of the compound primary key for 

292 this dimension's (logical) table, as well as references to their own 

293 tables. The ``required`` parameter does not include ``self`` (it 

294 can't, of course), but the corresponding attribute does - it is added 

295 by the constructor. 

296 implied : `NamedValueAbstractSet` [ `Dimension` ] 

297 Other dimensions whose keys are included in this dimension's (logical) 

298 table as foreign keys. 

299 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

300 Field specifications for all non-key fields in this dimension's table. 

301 uniqueKeys : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

302 Fields that can each be used to uniquely identify this dimension (given 

303 values for all required dimensions). The first of these is used as 

304 (part of) this dimension's table's primary key, while others are used 

305 to define unique constraints. 

306 

307 Notes 

308 ----- 

309 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

310 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

311 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

312 members. 

313 """ 

314 

315 def __init__( 

316 self, 

317 name: str, 

318 storage: dict, 

319 *, 

320 required: NamedValueSet[Dimension], 

321 implied: NamedValueAbstractSet[Dimension], 

322 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

323 uniqueKeys: NamedValueAbstractSet[ddl.FieldSpec], 

324 ): 

325 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

326 required.add(self) 

327 self._required = required.freeze() 

328 self._uniqueKeys = uniqueKeys 

329 

330 @property 

331 def required(self) -> NamedValueAbstractSet[Dimension]: 

332 # Docstring inherited from DimensionElement. 

333 return self._required 

334 

335 @property 

336 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

337 # Docstring inherited from Dimension. 

338 return self._uniqueKeys 

339 

340 

341class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

342 """A combination class that maps directly to a database table or query. 

343 

344 Parameters 

345 ---------- 

346 name : `str` 

347 Name of the dimension. 

348 storage : `dict` 

349 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

350 that will back this element in the registry (in a "cls" key) along 

351 with any other construction keyword arguments (in other keys). 

352 required : `NamedValueAbstractSet` [ `Dimension` ] 

353 Dimensions whose keys define the compound primary key for this 

354 combinations's (logical) table, as well as references to their own 

355 tables. 

356 implied : `NamedValueAbstractSet` [ `Dimension` ] 

357 Dimensions whose keys are included in this combinations's (logical) 

358 table as foreign keys. 

359 metadata : `NamedValueAbstractSet` [ `ddl.FieldSpec` ] 

360 Field specifications for all non-key fields in this combination's 

361 table. 

362 alwaysJoin : `bool`, optional 

363 If `True`, always include this element in any query or data ID in 

364 which its ``required`` dimensions appear, because it defines a 

365 relationship between those dimensions that must always be satisfied. 

366 

367 Notes 

368 ----- 

369 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

370 but it is the responsibility of 

371 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

372 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

373 

374 This class has a lot in common with `DatabaseDimension`, but they are 

375 expected to diverge in future changes, and the only way to make them share 

376 method implementations would be via multiple inheritance. Given the 

377 trivial nature of all of those implementations, this does not seem worth 

378 the drawbacks (particularly the constraints it imposes on constructor 

379 signatures). 

380 """ 

381 

382 def __init__( 

383 self, 

384 name: str, 

385 storage: dict, 

386 *, 

387 required: NamedValueAbstractSet[Dimension], 

388 implied: NamedValueAbstractSet[Dimension], 

389 metadata: NamedValueAbstractSet[ddl.FieldSpec], 

390 alwaysJoin: bool, 

391 populated_by: Dimension | None, 

392 ): 

393 super().__init__(name, storage=storage, implied=implied, metadata=metadata) 

394 self._required = required 

395 self._alwaysJoin = alwaysJoin 

396 self._populated_by = populated_by 

397 

398 @property 

399 def required(self) -> NamedValueAbstractSet[Dimension]: 

400 # Docstring inherited from DimensionElement. 

401 return self._required 

402 

403 @property 

404 def alwaysJoin(self) -> bool: 

405 # Docstring inherited from DimensionElement. 

406 return self._alwaysJoin 

407 

408 @property 

409 def populated_by(self) -> Dimension | None: 

410 # Docstring inherited. 

411 return self._populated_by 

412 

413 

414class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

415 """Construction visitor for database dimension and dimension combination. 

416 

417 Specifically, a construction visitor for `DatabaseDimension` and 

418 `DatabaseDimensionCombination`. 

419 

420 Parameters 

421 ---------- 

422 name : `str` 

423 Name of the dimension. 

424 storage : `dict` 

425 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

426 that will back this element in the registry (in a "cls" key) along 

427 with any other construction keyword arguments (in other keys). 

428 required : `~collections.abc.Set` [ `Dimension` ] 

429 Names of dimensions whose keys define the compound primary key for this 

430 element's (logical) table, as well as references to their own 

431 tables. 

432 implied : `~collections.abc.Set` [ `Dimension` ] 

433 Names of dimension whose keys are included in this elements's 

434 (logical) table as foreign keys. 

435 metadata : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

436 Field specifications for all non-key fields in this element's table. 

437 uniqueKeys : `~collections.abc.Iterable` [ `ddl.FieldSpec` ] 

438 Fields that can each be used to uniquely identify this dimension (given 

439 values for all required dimensions). The first of these is used as 

440 (part of) this dimension's table's primary key, while others are used 

441 to define unique constraints. Should be empty for 

442 `DatabaseDimensionCombination` definitions. 

443 alwaysJoin : `bool`, optional 

444 If `True`, always include this element in any query or data ID in 

445 which its ``required`` dimensions appear, because it defines a 

446 relationship between those dimensions that must always be satisfied. 

447 Should only be provided when a `DimensionCombination` is being 

448 constructed. 

449 populated_by: `Dimension`, optional 

450 The dimension that this element's records are always inserted, 

451 exported, and imported alongside. 

452 """ 

453 

454 def __init__( 

455 self, 

456 name: str, 

457 storage: dict, 

458 required: set[str], 

459 implied: set[str], 

460 metadata: Iterable[ddl.FieldSpec] = (), 

461 uniqueKeys: Iterable[ddl.FieldSpec] = (), 

462 alwaysJoin: bool = False, 

463 populated_by: str | None = None, 

464 ): 

465 super().__init__(name) 

466 self._storage = storage 

467 self._required = required 

468 self._implied = implied 

469 self._metadata = NamedValueSet(metadata).freeze() 

470 self._uniqueKeys = NamedValueSet(uniqueKeys).freeze() 

471 self._alwaysJoin = alwaysJoin 

472 self._populated_by = populated_by 

473 

474 def hasDependenciesIn(self, others: Set[str]) -> bool: 

475 # Docstring inherited from DimensionConstructionVisitor. 

476 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

477 

478 def visit(self, builder: DimensionConstructionBuilder) -> None: 

479 # Docstring inherited from DimensionConstructionVisitor. 

480 # Expand required dependencies. 

481 for name in tuple(self._required): # iterate over copy 

482 self._required.update(builder.dimensions[name].required.names) 

483 # Transform required and implied Dimension names into instances, 

484 # and reorder to match builder's order. 

485 required: NamedValueSet[Dimension] = NamedValueSet() 

486 implied: NamedValueSet[Dimension] = NamedValueSet() 

487 for dimension in builder.dimensions: 

488 if dimension.name in self._required: 

489 required.add(dimension) 

490 if dimension.name in self._implied: 

491 implied.add(dimension) 

492 

493 if self._uniqueKeys: 

494 if self._alwaysJoin: 

495 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

496 # Special handling for creating Dimension instances. 

497 dimension = DatabaseDimension( 

498 self.name, 

499 storage=self._storage, 

500 required=required, 

501 implied=implied.freeze(), 

502 metadata=self._metadata, 

503 uniqueKeys=self._uniqueKeys, 

504 ) 

505 builder.dimensions.add(dimension) 

506 builder.elements.add(dimension) 

507 else: 

508 # Special handling for creating DimensionCombination instances. 

509 combination = DatabaseDimensionCombination( 

510 self.name, 

511 storage=self._storage, 

512 required=required, 

513 implied=implied.freeze(), 

514 metadata=self._metadata, 

515 alwaysJoin=self._alwaysJoin, 

516 populated_by=( 

517 builder.dimensions[self._populated_by] if self._populated_by is not None else None 

518 ), 

519 ) 

520 builder.elements.add(combination)