Coverage for python/lsst/daf/butler/dimensions/_database.py: 46%

148 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ( 

31 "DatabaseDimension", 

32 "DatabaseDimensionCombination", 

33 "DatabaseDimensionElement", 

34 "DatabaseTopologicalFamily", 

35) 

36 

37from collections.abc import Iterable, Mapping, Set 

38from types import MappingProxyType 

39from typing import TYPE_CHECKING 

40 

41from lsst.utils.classes import cached_getter 

42 

43from .._named import NamedValueAbstractSet, NamedValueSet 

44from .._topology import TopologicalFamily, TopologicalSpace 

45from ._elements import Dimension, DimensionCombination, DimensionElement, KeyColumnSpec, MetadataColumnSpec 

46from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

47 

48if TYPE_CHECKING: 

49 from ._governor import GovernorDimension 

50 from ._universe import DimensionUniverse 

51 

52 

53class DatabaseTopologicalFamily(TopologicalFamily): 

54 """Database topological family implementation. 

55 

56 A `TopologicalFamily` implementation for the `DatabaseDimension` and 

57 `DatabaseDimensionCombination` objects that have direct database 

58 representations. 

59 

60 Parameters 

61 ---------- 

62 name : `str` 

63 Name of the family. 

64 space : `TopologicalSpace` 

65 Space in which this family's regions live. 

66 members : `NamedValueAbstractSet` [ `DimensionElement` ] 

67 The members of this family, ordered according to the priority used 

68 in `choose` (first-choice member first). 

69 """ 

70 

71 def __init__( 

72 self, 

73 name: str, 

74 space: TopologicalSpace, 

75 *, 

76 members: NamedValueAbstractSet[DimensionElement], 

77 ): 

78 super().__init__(name, space) 

79 self.members = members 

80 

81 def choose(self, endpoints: Set[str], universe: DimensionUniverse) -> DimensionElement: 

82 # Docstring inherited from TopologicalFamily. 

83 for member in self.members: 

84 if member.name in endpoints: 

85 return member 

86 raise RuntimeError(f"No recognized endpoints for {self.name} in {endpoints}.") 

87 

88 @property 

89 @cached_getter 

90 def governor(self) -> GovernorDimension: 

91 """Return `GovernorDimension` common to all members of this family. 

92 

93 (`GovernorDimension`). 

94 """ 

95 governors = {m.governor for m in self.members} 

96 if None in governors: 

97 raise RuntimeError( 

98 f"Bad {self.space.name} family definition {self.name}: at least one member " 

99 f"in {self.members} has no GovernorDimension dependency." 

100 ) 

101 try: 

102 (result,) = governors 

103 except ValueError: 

104 raise RuntimeError( 

105 f"Bad {self.space.name} family definition {self.name}: multiple governors {governors} " 

106 f"in {self.members}." 

107 ) from None 

108 return result # type: ignore 

109 

110 members: NamedValueAbstractSet[DimensionElement] 

111 """The members of this family, ordered according to the priority used in 

112 `choose` (first-choice member first). 

113 """ 

114 

115 

116class DatabaseTopologicalFamilyConstructionVisitor(DimensionConstructionVisitor): 

117 """A construction visitor for `DatabaseTopologicalFamily`. 

118 

119 This visitor depends on (and is thus visited after) its members. 

120 

121 Parameters 

122 ---------- 

123 name : `str` 

124 Name of the family. 

125 space : `TopologicalSpace` 

126 Space in which this family's regions live. 

127 members : `~collections.abc.Iterable` [ `str` ] 

128 The names of the members of this family, ordered according to the 

129 priority used in `choose` (first-choice member first). 

130 """ 

131 

132 def __init__(self, name: str, space: TopologicalSpace, members: Iterable[str]): 

133 super().__init__(name) 

134 self._space = space 

135 self._members = tuple(members) 

136 

137 def hasDependenciesIn(self, others: Set[str]) -> bool: 

138 # Docstring inherited from DimensionConstructionVisitor. 

139 return not others.isdisjoint(self._members) 

140 

141 def visit(self, builder: DimensionConstructionBuilder) -> None: 

142 # Docstring inherited from DimensionConstructionVisitor. 

143 members = NamedValueSet(builder.elements[name] for name in self._members) 

144 family = DatabaseTopologicalFamily(self.name, self._space, members=members.freeze()) 

145 builder.topology[self._space].add(family) 

146 for member in members: 

147 assert isinstance(member, DatabaseDimension | DatabaseDimensionCombination) 

148 other = member._topology.setdefault(self._space, family) 

149 if other is not family: 

150 raise RuntimeError( 

151 f"{member.name} is declared to be a member of (at least) two " 

152 f"{self._space.name} families: {other.name} and {family.name}." 

153 ) 

154 

155 

156class DatabaseDimensionElement(DimensionElement): 

157 """An intermediate base class for `DimensionElement` database classes. 

158 

159 Theese classes are ones whose instances map directly to a database 

160 table or query. 

161 

162 Parameters 

163 ---------- 

164 name : `str` 

165 Name of the dimension. 

166 storage : `dict` 

167 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

168 that will back this element in the registry (in a "cls" key) along 

169 with any other construction keyword arguments (in other keys). 

170 implied : `NamedValueAbstractSet` [ `Dimension` ] 

171 Other dimensions whose keys are included in this dimension's (logical) 

172 table as foreign keys. 

173 metadata_columns : `NamedValueAbstractSet` [ `MetadataColumnSpec` ] 

174 Field specifications for all non-key fields in this dimension's table. 

175 doc : `str` 

176 Extended description of this element. 

177 """ 

178 

179 def __init__( 

180 self, 

181 name: str, 

182 storage: dict, 

183 *, 

184 implied: NamedValueAbstractSet[Dimension], 

185 metadata_columns: NamedValueAbstractSet[MetadataColumnSpec], 

186 doc: str, 

187 ): 

188 self._name = name 

189 self._storage = storage 

190 self._implied = implied 

191 self._metadata_columns = metadata_columns 

192 self._topology: dict[TopologicalSpace, DatabaseTopologicalFamily] = {} 

193 self._doc = doc 

194 

195 @property 

196 def name(self) -> str: 

197 # Docstring inherited from TopologicalRelationshipEndpoint. 

198 return self._name 

199 

200 @property 

201 def implied(self) -> NamedValueAbstractSet[Dimension]: 

202 # Docstring inherited from DimensionElement. 

203 return self._implied 

204 

205 @property 

206 def metadata_columns(self) -> NamedValueAbstractSet[MetadataColumnSpec]: 

207 # Docstring inherited from DimensionElement. 

208 return self._metadata_columns 

209 

210 @property 

211 def implied_union_target(self) -> DimensionElement | None: 

212 # Docstring inherited from DimensionElement. 

213 # This is a bit encapsulation-breaking, but it'll all be cleaned up 

214 # soon when we get rid of the storage objects entirely. 

215 storage = self._storage.get("nested") 

216 if storage is None: 

217 storage = self._storage 

218 name = storage.get("view_of") 

219 return self.universe[name] if name is not None else None 

220 

221 @property 

222 def is_cached(self) -> bool: 

223 # Docstring inherited. 

224 # This is a bit encapsulation-breaking, but it'll all be cleaned up 

225 # soon when we get rid of the storage objects entirely. 

226 return "caching" in self._storage["cls"] 

227 

228 @property 

229 def documentation(self) -> str: 

230 # Docstring inherited from DimensionElement. 

231 return self._doc 

232 

233 @property 

234 def topology(self) -> Mapping[TopologicalSpace, DatabaseTopologicalFamily]: 

235 # Docstring inherited from TopologicalRelationshipEndpoint 

236 return MappingProxyType(self._topology) 

237 

238 @property 

239 def spatial(self) -> DatabaseTopologicalFamily | None: 

240 # Docstring inherited from TopologicalRelationshipEndpoint 

241 return self.topology.get(TopologicalSpace.SPATIAL) 

242 

243 @property 

244 def temporal(self) -> DatabaseTopologicalFamily | None: 

245 # Docstring inherited from TopologicalRelationshipEndpoint 

246 return self.topology.get(TopologicalSpace.TEMPORAL) 

247 

248 

249class DatabaseDimension(Dimension, DatabaseDimensionElement): 

250 """A `Dimension` class that maps directly to a database table or query. 

251 

252 Parameters 

253 ---------- 

254 name : `str` 

255 Name of the dimension. 

256 storage : `dict` 

257 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

258 that will back this element in the registry (in a "cls" key) along 

259 with any other construction keyword arguments (in other keys). 

260 required : `NamedValueSet` [ `Dimension` ] 

261 Other dimensions whose keys are part of the compound primary key for 

262 this dimension's (logical) table, as well as references to their own 

263 tables. The ``required`` parameter does not include ``self`` (it 

264 can't, of course), but the corresponding attribute does - it is added 

265 by the constructor. 

266 implied : `NamedValueAbstractSet` [ `Dimension` ] 

267 Other dimensions whose keys are included in this dimension's (logical) 

268 table as foreign keys. 

269 metadata_columns : `NamedValueAbstractSet` [ `MetadataColumnSpec` ] 

270 Field specifications for all non-key fields in this dimension's table. 

271 unique_keys : `NamedValueAbstractSet` [ `KeyColumnSpec` ] 

272 Fields that can each be used to uniquely identify this dimension (given 

273 values for all required dimensions). The first of these is used as 

274 (part of) this dimension's table's primary key, while others are used 

275 to define unique constraints. 

276 doc : `str` 

277 Extended description of this element. 

278 

279 Notes 

280 ----- 

281 `DatabaseDimension` objects may belong to a `TopologicalFamily`, but it is 

282 the responsibility of `DatabaseTopologicalFamilyConstructionVisitor` to 

283 update the `~TopologicalRelationshipEndpoint.topology` attribute of their 

284 members. 

285 """ 

286 

287 def __init__( 

288 self, 

289 name: str, 

290 storage: dict, 

291 *, 

292 required: NamedValueSet[Dimension], 

293 implied: NamedValueAbstractSet[Dimension], 

294 metadata_columns: NamedValueAbstractSet[MetadataColumnSpec], 

295 unique_keys: NamedValueAbstractSet[KeyColumnSpec], 

296 doc: str, 

297 ): 

298 super().__init__(name, storage=storage, implied=implied, metadata_columns=metadata_columns, doc=doc) 

299 required.add(self) 

300 self._required = required.freeze() 

301 self._unique_keys = unique_keys 

302 

303 @property 

304 def required(self) -> NamedValueAbstractSet[Dimension]: 

305 # Docstring inherited from DimensionElement. 

306 return self._required 

307 

308 @property 

309 def unique_keys(self) -> NamedValueAbstractSet[KeyColumnSpec]: 

310 # Docstring inherited from Dimension. 

311 return self._unique_keys 

312 

313 

314class DatabaseDimensionCombination(DimensionCombination, DatabaseDimensionElement): 

315 """A combination class that maps directly to a database table or query. 

316 

317 Parameters 

318 ---------- 

319 name : `str` 

320 Name of the dimension. 

321 storage : `dict` 

322 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

323 that will back this element in the registry (in a "cls" key) along 

324 with any other construction keyword arguments (in other keys). 

325 required : `NamedValueAbstractSet` [ `Dimension` ] 

326 Dimensions whose keys define the compound primary key for this 

327 combinations's (logical) table, as well as references to their own 

328 tables. 

329 implied : `NamedValueAbstractSet` [ `Dimension` ] 

330 Dimensions whose keys are included in this combinations's (logical) 

331 table as foreign keys. 

332 metadata_columns : `NamedValueAbstractSet` [ `MetadataColumnSpec` ] 

333 Field specifications for all non-key fields in this combination's 

334 table. 

335 alwaysJoin : `bool`, optional 

336 If `True`, always include this element in any query or data ID in 

337 which its ``required`` dimensions appear, because it defines a 

338 relationship between those dimensions that must always be satisfied. 

339 populated_by : `Dimension` or `None` 

340 The dimension that this element's records are always inserted, 

341 exported, and imported alongside. 

342 doc : `str` 

343 Extended description of this element. 

344 

345 Notes 

346 ----- 

347 `DatabaseDimensionCombination` objects may belong to a `TopologicalFamily`, 

348 but it is the responsibility of 

349 `DatabaseTopologicalFamilyConstructionVisitor` to update the 

350 `~TopologicalRelationshipEndpoint.topology` attribute of their members. 

351 

352 This class has a lot in common with `DatabaseDimension`, but they are 

353 expected to diverge in future changes, and the only way to make them share 

354 method implementations would be via multiple inheritance. Given the 

355 trivial nature of all of those implementations, this does not seem worth 

356 the drawbacks (particularly the constraints it imposes on constructor 

357 signatures). 

358 """ 

359 

360 def __init__( 

361 self, 

362 name: str, 

363 storage: dict, 

364 *, 

365 required: NamedValueAbstractSet[Dimension], 

366 implied: NamedValueAbstractSet[Dimension], 

367 metadata_columns: NamedValueAbstractSet[MetadataColumnSpec], 

368 alwaysJoin: bool, 

369 populated_by: Dimension | None, 

370 doc: str, 

371 ): 

372 super().__init__(name, storage=storage, implied=implied, metadata_columns=metadata_columns, doc=doc) 

373 self._required = required 

374 self._alwaysJoin = alwaysJoin 

375 self._populated_by = populated_by 

376 

377 @property 

378 def required(self) -> NamedValueAbstractSet[Dimension]: 

379 # Docstring inherited from DimensionElement. 

380 return self._required 

381 

382 @property 

383 def alwaysJoin(self) -> bool: 

384 # Docstring inherited from DimensionElement. 

385 return self._alwaysJoin 

386 

387 @property 

388 def defines_relationships(self) -> bool: 

389 # Docstring inherited from DimensionElement. 

390 return self._alwaysJoin or bool(self.implied) 

391 

392 @property 

393 def populated_by(self) -> Dimension | None: 

394 # Docstring inherited. 

395 return self._populated_by 

396 

397 

398class DatabaseDimensionElementConstructionVisitor(DimensionConstructionVisitor): 

399 """Construction visitor for database dimension and dimension combination. 

400 

401 Specifically, a construction visitor for `DatabaseDimension` and 

402 `DatabaseDimensionCombination`. 

403 

404 Parameters 

405 ---------- 

406 name : `str` 

407 Name of the dimension. 

408 storage : `dict` 

409 Fully qualified name of the `DatabaseDimensionRecordStorage` subclass 

410 that will back this element in the registry (in a "cls" key) along 

411 with any other construction keyword arguments (in other keys). 

412 required : `~collections.abc.Set` [ `Dimension` ] 

413 Names of dimensions whose keys define the compound primary key for this 

414 element's (logical) table, as well as references to their own 

415 tables. 

416 implied : `~collections.abc.Set` [ `Dimension` ] 

417 Names of dimension whose keys are included in this elements's 

418 (logical) table as foreign keys. 

419 doc : `str` 

420 Extended description of this element. 

421 metadata_columns : `~collections.abc.Iterable` [ `MetadataColumnSpec` ] 

422 Field specifications for all non-key fields in this element's table. 

423 unique_keys : `~collections.abc.Iterable` [ `KeyColumnSpec` ] 

424 Fields that can each be used to uniquely identify this dimension (given 

425 values for all required dimensions). The first of these is used as 

426 (part of) this dimension's table's primary key, while others are used 

427 to define unique constraints. Should be empty for 

428 `DatabaseDimensionCombination` definitions. 

429 alwaysJoin : `bool`, optional 

430 If `True`, always include this element in any query or data ID in 

431 which its ``required`` dimensions appear, because it defines a 

432 relationship between those dimensions that must always be satisfied. 

433 Should only be provided when a `DimensionCombination` is being 

434 constructed. 

435 populated_by : `Dimension`, optional 

436 The dimension that this element's records are always inserted, 

437 exported, and imported alongside. 

438 """ 

439 

440 def __init__( 

441 self, 

442 name: str, 

443 storage: dict, 

444 required: set[str], 

445 implied: set[str], 

446 doc: str, 

447 metadata_columns: Iterable[MetadataColumnSpec] = (), 

448 unique_keys: Iterable[KeyColumnSpec] = (), 

449 alwaysJoin: bool = False, 

450 populated_by: str | None = None, 

451 ): 

452 super().__init__(name) 

453 self._storage = storage 

454 self._required = required 

455 self._implied = implied 

456 self._metadata_columns = NamedValueSet(metadata_columns).freeze() 

457 self._unique_keys = NamedValueSet(unique_keys).freeze() 

458 self._alwaysJoin = alwaysJoin 

459 self._populated_by = populated_by 

460 self._doc = doc 

461 

462 def hasDependenciesIn(self, others: Set[str]) -> bool: 

463 # Docstring inherited from DimensionConstructionVisitor. 

464 return not (self._required.isdisjoint(others) and self._implied.isdisjoint(others)) 

465 

466 def visit(self, builder: DimensionConstructionBuilder) -> None: 

467 # Docstring inherited from DimensionConstructionVisitor. 

468 # Expand required dependencies. 

469 for name in tuple(self._required): # iterate over copy 

470 self._required.update(builder.dimensions[name].required.names) 

471 # Transform required and implied Dimension names into instances, 

472 # and reorder to match builder's order. 

473 required: NamedValueSet[Dimension] = NamedValueSet() 

474 implied: NamedValueSet[Dimension] = NamedValueSet() 

475 for dimension in builder.dimensions: 

476 if dimension.name in self._required: 

477 required.add(dimension) 

478 if dimension.name in self._implied: 

479 implied.add(dimension) 

480 

481 if self._unique_keys: 

482 if self._alwaysJoin: 

483 raise RuntimeError(f"'alwaysJoin' is not a valid option for Dimension object {self.name}.") 

484 # Special handling for creating Dimension instances. 

485 dimension = DatabaseDimension( 

486 self.name, 

487 storage=self._storage, 

488 required=required, 

489 implied=implied.freeze(), 

490 metadata_columns=self._metadata_columns, 

491 unique_keys=self._unique_keys, 

492 doc=self._doc, 

493 ) 

494 builder.dimensions.add(dimension) 

495 builder.elements.add(dimension) 

496 else: 

497 # Special handling for creating DimensionCombination instances. 

498 combination = DatabaseDimensionCombination( 

499 self.name, 

500 storage=self._storage, 

501 required=required, 

502 implied=implied.freeze(), 

503 doc=self._doc, 

504 metadata_columns=self._metadata_columns, 

505 alwaysJoin=self._alwaysJoin, 

506 populated_by=( 

507 builder.dimensions[self._populated_by] if self._populated_by is not None else None 

508 ), 

509 ) 

510 builder.elements.add(combination)