Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"] 

25 

26from typing import ( 

27 Any, 

28 AbstractSet, 

29 Dict, 

30 Iterable, 

31 Optional, 

32 Set, 

33 Type, 

34 TYPE_CHECKING, 

35) 

36 

37from sqlalchemy import BigInteger 

38 

39from lsst.sphgeom import Pixelization 

40from ..utils import immutable 

41from ..named import NamedValueSet 

42from .. import ddl 

43 

44if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from .universe import DimensionUniverse 

46 from .graph import DimensionGraph 

47 from .records import DimensionRecord 

48 

49 

50class RelatedDimensions: 

51 """A semi-internal struct containing the names of the dimension elements 

52 related to the one holding the instance of this struct. 

53 

54 This object is used as the type of `DimensionElement._related`, which is 

55 considered private _to the dimensions subpackage_, rather that private or 

56 protected within `DimensionElement` itself. 

57 

58 Parameters 

59 ---------- 

60 required : `set` [ `str` ] 

61 The names of other dimensions that are used to form the (compound) 

62 primary key for this element, as well as foreign keys. 

63 implied : `set` [ `str` ] 

64 The names of other dimensions that are used to define foreign keys 

65 for this element, but not primary keys. 

66 spatial : `str`, optional 

67 The name of a `DimensionElement` whose spatial regions this element's 

68 region aggregates, or the name of this element if it has a region 

69 that is not an aggregate. `None` (default) if this element is not 

70 associated with region. 

71 temporal : `str`, optional 

72 The name of a `DimensionElement` whose timespans this element's 

73 timespan aggregates, or the name of this element if it has a timespan 

74 that is not an aggregate. `None` (default) if this element is not 

75 associated with timespan. 

76 """ 

77 def __init__(self, required: Set[str], implied: Set[str], 

78 spatial: Optional[str] = None, temporal: Optional[str] = None): 

79 self.required = required 

80 self.implied = implied 

81 self.spatial = spatial 

82 self.temporal = temporal 

83 self.dependencies = set(self.required | self.implied) 

84 

85 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies") 

86 

87 def expand(self, universe: DimensionUniverse) -> None: 

88 """Expand ``required`` and ``dependencies`` recursively. 

89 

90 Parameters 

91 ---------- 

92 universe : `DimensionUniverse` 

93 Object containing all other dimension elements. 

94 """ 

95 for req in tuple(self.required): 

96 other = universe[req]._related 

97 self.required.update(other.required) 

98 self.dependencies.update(other.dependencies) 

99 for dep in self.implied: 

100 other = universe[dep]._related 

101 self.dependencies.update(other.dependencies) 

102 

103 required: Set[str] 

104 """The names of dimensions that are used to form the (compound) primary key 

105 for this element, as well as foreign keys. 

106 

107 For true `Dimension` instances, this should be constructed without the 

108 dimension's own name, with the `Dimension` itself adding it later (after 

109 `dependencies` is defined). 

110 """ 

111 

112 implied: Set[str] 

113 """The names of other dimensions that are used to define foreign keys for 

114 this element, but not primary keys. 

115 """ 

116 

117 dependencies: Set[str] 

118 """The names of all dimensions in `required` or `implied`. 

119 

120 Immediately after construction, this is equal to the union of `required` 

121 `implied`. After `expand` is called, this may not be true, as this will 

122 include the required and implied dependencies (recursively) of implied 

123 dependencies, while `implied` is not expanded recursively. 

124 

125 For true `Dimension` instances, this never includes the dimension's own 

126 name. 

127 """ 

128 

129 spatial: Optional[str] 

130 """The name of a dimension element to which this element delegates spatial 

131 region handling, if any (see `DimensionElement.spatial`). 

132 """ 

133 

134 temporal: Optional[str] 

135 """The name of a dimension element to which this element delegates 

136 timespan handling, if any (see `DimensionElement.temporal`). 

137 """ 

138 

139 

140@immutable 

141class DimensionElement: 

142 """A named data-organization concept that defines a label and/or metadata 

143 in the dimensions system. 

144 

145 A `DimensionElement` instance typically corresponds to a table in the 

146 `Registry`; the rows in that table are represented by instances of a 

147 `DimensionRecord` subclass. Most `DimensionElement` instances are 

148 instances of its `Dimension` subclass, which is used for elements that can 

149 be used as data ID keys. The base class itself can be used for other 

150 dimension tables that provide metadata keyed by true dimensions. 

151 

152 Parameters 

153 ---------- 

154 name : `str` 

155 Name of the element. Used as at least part of the table name, if 

156 the dimension in associated with a database table. 

157 related : `RelatedDimensions` 

158 Struct containing the names of related dimensions. 

159 metadata : iterable of `FieldSpec` 

160 Additional metadata fields included in this element's table. 

161 cached : `bool` 

162 Whether `Registry` should cache records of this element in-memory. 

163 viewOf : `str`, optional 

164 Name of another table this element's records should be drawn from. The 

165 fields of this table must be a superset of the fields of the element. 

166 

167 Notes 

168 ----- 

169 `DimensionElement` instances should always be constructed by and retreived 

170 from a `DimensionUniverse`. They are immutable after they are fully 

171 constructed, and should never be copied. 

172 

173 Pickling a `DimensionElement` just records its name and universe; 

174 unpickling one actually just looks up the element via the singleton 

175 dictionary of all universes. This allows pickle to be used to transfer 

176 elements between processes, but only when each process initializes its own 

177 instance of the same `DimensionUniverse`. 

178 """ 

179 

180 def __init__(self, name: str, *, 

181 related: RelatedDimensions, 

182 metadata: Iterable[ddl.FieldSpec] = (), 

183 cached: bool = False, 

184 viewOf: Optional[str] = None, 

185 alwaysJoin: bool = False): 

186 self.name = name 

187 self._related = related 

188 self.metadata = NamedValueSet(metadata) 

189 self.metadata.freeze() 

190 self.cached = cached 

191 self.viewOf = viewOf 

192 self.alwaysJoin = alwaysJoin 

193 

194 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]) -> None: 

195 """Finish construction of the element and add it to the given universe. 

196 

197 For internal use by `DimensionUniverse` only. 

198 

199 Parameters 

200 ---------- 

201 universe : `DimensionUniverse` 

202 The under-construction dimension universe. It can be relied upon 

203 to contain all dimensions that this element requires or implies. 

204 elementsToDo : `dict` [ `str`, `DimensionElement` ] 

205 A dictionary containing all dimension elements that have not yet 

206 been added to ``universe``, keyed by name. 

207 """ 

208 # Attach set self.universe and add self to universe attributes; 

209 # let subclasses override which attributes by calling a separate 

210 # method. 

211 self._attachToUniverse(universe) 

212 # Expand dependencies. 

213 self._related.expand(universe) 

214 # Define public DimensionElement versions of some of the name sets 

215 # in self._related. 

216 self.required = NamedValueSet(universe.sorted(self._related.required)) 

217 self.required.freeze() 

218 self.implied = NamedValueSet(universe.sorted(self._related.implied)) 

219 self.implied.freeze() 

220 self.dimensions = NamedValueSet(self.required) 

221 for d in self.implied: 

222 self.dimensions.add(d) 

223 self.dimensions.freeze() 

224 assert all(a == b for a, b in zip(self.required, self.dimensions)), \ 

225 "NamedValueSet.add is expected to preserve order." 

226 # Set self.spatial and self.temporal to DimensionElement instances from 

227 # the private _related versions of those. 

228 for s in ("spatial", "temporal"): 

229 targetName = getattr(self._related, s, None) 

230 if targetName is None: 

231 target = None 

232 elif targetName == self.name: 

233 target = self 

234 else: 

235 target = universe.get(targetName) 

236 if target is None: 

237 try: 

238 target = elementsToDo[targetName] 

239 except KeyError as err: 

240 raise LookupError( 

241 f"Could not find {s} provider {targetName} for {self.name}." 

242 ) from err 

243 setattr(self, s, target) 

244 # Attach a DimensionGraph that provides the public API for getting 

245 # at requirements. Again delegate to subclasses. 

246 self._attachGraph() 

247 # Create and attach a DimensionRecord subclass to hold values of this 

248 # dimension type. 

249 from .records import _subclassDimensionRecord 

250 self.RecordClass = _subclassDimensionRecord(self) 

251 

252 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

253 """Add the element to the given universe. 

254 

255 Called only by `_finish`, but may be overridden by subclasses. 

256 """ 

257 self.universe = universe 

258 self.universe._elements.add(self) 

259 

260 def _attachGraph(self) -> None: 

261 """Initialize the `graph` attribute for this element. 

262 

263 Called only by `_finish`, but may be overridden by subclasses. 

264 """ 

265 from .graph import DimensionGraph 

266 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False) 

267 

268 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

269 """Return `True` if this element should be included in `DimensionGraph` 

270 that includes the named dimensions. 

271 

272 For internal use by `DimensionGraph` only. 

273 """ 

274 return self._related.required.issubset(dimensionNames) 

275 

276 def __str__(self) -> str: 

277 return self.name 

278 

279 def __repr__(self) -> str: 

280 return f"{type(self).__name__}({self.name})" 

281 

282 def __eq__(self, other: Any) -> bool: 

283 try: 

284 return self.name == other.name 

285 except AttributeError: 

286 return self.name == other 

287 

288 def __hash__(self) -> int: 

289 return hash(self.name) 

290 

291 def __lt__(self, other: DimensionElement) -> bool: 

292 try: 

293 return self.universe.getElementIndex(self.name) < self.universe.getElementIndex(other.name) 

294 except KeyError: 

295 return NotImplemented 

296 

297 def __le__(self, other: DimensionElement) -> bool: 

298 try: 

299 return self.universe.getElementIndex(self.name) <= self.universe.getElementIndex(other.name) 

300 except KeyError: 

301 return NotImplemented 

302 

303 def __gt__(self, other: DimensionElement) -> bool: 

304 try: 

305 return self.universe.getElementIndex(self.name) > self.universe.getElementIndex(other.name) 

306 except KeyError: 

307 return NotImplemented 

308 

309 def __ge__(self, other: DimensionElement) -> bool: 

310 try: 

311 return self.universe.getElementIndex(self.name) >= self.universe.getElementIndex(other.name) 

312 except KeyError: 

313 return NotImplemented 

314 

315 def hasTable(self) -> bool: 

316 """Return `True` if this element is associated with a table 

317 (even if that table "belongs" to another element). 

318 

319 Instances of the `DimensionElement` base class itself are always 

320 associated with tables. 

321 """ 

322 return True 

323 

324 @classmethod 

325 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

326 """Callable used for unpickling. 

327 

328 For internal use only. 

329 """ 

330 return universe[name] 

331 

332 def __reduce__(self) -> tuple: 

333 return (self._unpickle, (self.universe, self.name)) 

334 

335 # Class attributes below are shadowed by instance attributes, and are 

336 # present just to hold the docstrings for those instance attributes. 

337 

338 universe: DimensionUniverse 

339 """The universe of all compatible dimensions with which this element is 

340 associated (`DimensionUniverse`). 

341 """ 

342 

343 name: str 

344 """Unique name for this dimension element (`str`). 

345 """ 

346 

347 graph: DimensionGraph 

348 """Minimal graph that includes this element (`DimensionGraph`). 

349 

350 ``self.graph.required`` includes all dimensions whose primary key values 

351 are sufficient (often necessary) to uniquely identify ``self`` 

352 (including ``self` if ``isinstance(self, Dimension)``. 

353 ``self.graph.implied`` includes all dimensions also identified (possibly 

354 recursively) by this set. 

355 """ 

356 

357 required: NamedValueSet[Dimension] 

358 """Dimensions that are sufficient (often necessary) to uniquely identify 

359 a record of this dimension element. 

360 

361 For elements with a database representation, these dimension are exactly 

362 those used to form the (possibly compound) primary key, and all dimensions 

363 here that are not ``self`` are also used to form foreign keys. 

364 

365 For `Dimension` instances, this should be exactly the same as 

366 ``graph.required``, but that may not be true for `DimensionElement` 

367 instances in general. When they differ, there are multiple combinations 

368 of dimensions that uniquely identify this element, but this one is more 

369 direct. 

370 """ 

371 

372 implied: NamedValueSet[Dimension] 

373 """Other dimensions that are uniquely identified directly by a record of 

374 this dimension element. 

375 

376 For elements with a database representation, these are exactly the 

377 dimensions used to form foreign key constraints whose fields are not 

378 (wholly) also part of the primary key. 

379 

380 Unlike ``self.graph.implied``, this set is not expanded recursively. 

381 """ 

382 

383 dimensions: NamedValueSet[Dimension] 

384 """The union of `required` and `implied`, with all elements in `required` 

385 before any elements in `implied`. 

386 

387 This differs from ``self.graph.dimension`` both in order and in content: 

388 

389 - as in ``self.implied``, implied dimensions are not expanded recursively 

390 here; 

391 - implied dimensions appear after required dimensions here, instead of 

392 being topologically ordered. 

393 

394 As a result, this set is ordered consistently with 

395 ``self.RecordClass.fields``. 

396 """ 

397 

398 spatial: Optional[DimensionElement] 

399 """A `DimensionElement` whose spatial regions this element's region 

400 aggregates, or ``self`` if it has a region that is not an aggregate 

401 (`DimensionElement` or `None`). 

402 """ 

403 

404 temporal: Optional[DimensionElement] 

405 """A `DimensionElement` whose timespans this element's timespan 

406 aggregates, or ``self`` if it has a timespan that is not an aggregate 

407 (`DimensionElement` or `None`). 

408 """ 

409 

410 metadata: NamedValueSet[ddl.FieldSpec] 

411 """Additional metadata fields included in this element's table 

412 (`NamedValueSet` of `FieldSpec`). 

413 """ 

414 

415 RecordClass: Type[DimensionRecord] 

416 """The `DimensionRecord` subclass used to hold records for this element 

417 (`type`). 

418 

419 Because `DimensionRecord` subclasses are generated dynamically, this type 

420 cannot be imported directly and hence canonly be obtained from this 

421 attribute. 

422 """ 

423 

424 cached: bool 

425 """Whether `Registry` should cache records of this element in-memory 

426 (`bool`). 

427 """ 

428 

429 viewOf: Optional[str] 

430 """Name of another table this elements records are drawn from (`str` or 

431 `None`). 

432 """ 

433 

434 

435@immutable 

436class Dimension(DimensionElement): 

437 """A named data-organization concept that can be used as a key in a data 

438 ID. 

439 

440 Parameters 

441 ---------- 

442 name : `str` 

443 Name of the dimension. Used as at least part of the table name, if 

444 the dimension in associated with a database table, and as an alternate 

445 key (instead of the instance itself) in data IDs. 

446 related : `RelatedDimensions` 

447 Struct containing the names of related dimensions. 

448 uniqueKeys : iterable of `FieldSpec` 

449 Fields that can *each* be used to uniquely identify this dimension 

450 (once all required dependencies are also identified). The first entry 

451 will be used as the table's primary key and as a foriegn key field in 

452 the fields of dependent tables. 

453 kwds 

454 Additional keyword arguments are forwarded to the `DimensionElement` 

455 constructor. 

456 """ 

457 

458 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec], 

459 **kwargs: Any): 

460 related.required.add(name) 

461 super().__init__(name, related=related, **kwargs) 

462 self.uniqueKeys = NamedValueSet(uniqueKeys) 

463 self.uniqueKeys.freeze() 

464 self.primaryKey, *alternateKeys = uniqueKeys 

465 self.alternateKeys = NamedValueSet(alternateKeys) 

466 self.alternateKeys.freeze() 

467 

468 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

469 # Docstring inherited from DimensionElement._attachToUniverse. 

470 super()._attachToUniverse(universe) 

471 universe._dimensions.add(self) 

472 

473 def _attachGraph(self) -> None: 

474 # Docstring inherited from DimensionElement._attachGraph. 

475 from .graph import DimensionGraph 

476 self.graph = DimensionGraph(self.universe, 

477 names=self._related.dependencies.union([self.name]), 

478 conform=False) 

479 

480 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

481 # Docstring inherited from DimensionElement._shouldBeInGraph. 

482 return self.name in dimensionNames 

483 

484 # Class attributes below are shadowed by instance attributes, and are 

485 # present just to hold the docstrings for those instance attributes. 

486 

487 uniqueKeys: NamedValueSet[ddl.FieldSpec] 

488 """All fields that can individually be used to identify records of this 

489 element, given the primary keys of all required dependencies 

490 (`NamedValueSet` of `FieldSpec`). 

491 """ 

492 

493 primaryKey: ddl.FieldSpec 

494 """The primary key field for this dimension (`FieldSpec`). 

495 

496 Note that the database primary keys for dimension tables are in general 

497 compound; this field is the only field in the database primary key that is 

498 not also a foreign key (to a required dependency dimension table). 

499 """ 

500 

501 alternateKeys: NamedValueSet[ddl.FieldSpec] 

502 """Additional unique key fields for this dimension that are not the the 

503 primary key (`NamedValueSet` of `FieldSpec`). 

504 """ 

505 

506 

507@immutable 

508class SkyPixDimension(Dimension): 

509 """A special `Dimension` subclass for hierarchical pixelizations of the 

510 sky. 

511 

512 Unlike most other dimensions, skypix dimension records are not stored in 

513 the database, as these records only contain an integer pixel ID and a 

514 region on the sky, and each of these can be computed directly from the 

515 other. 

516 

517 Parameters 

518 ---------- 

519 name : `str` 

520 Name of the dimension. By convention, this is a lowercase string 

521 abbreviation for the pixelization followed by its integer level, 

522 such as "htm7". 

523 pixelization : `sphgeom.Pixelization` 

524 Pixelization instance that can compute regions from IDs and IDs from 

525 points. 

526 """ 

527 

528 def __init__(self, name: str, pixelization: Pixelization): 

529 related = RelatedDimensions(required=set(), implied=set(), spatial=name) 

530 uniqueKeys = [ddl.FieldSpec(name="id", dtype=BigInteger, primaryKey=True, nullable=False)] 

531 super().__init__(name, related=related, uniqueKeys=uniqueKeys) 

532 self.pixelization = pixelization 

533 

534 def hasTable(self) -> bool: 

535 # Docstring inherited from DimensionElement.hasTable. 

536 return False 

537 

538 # Class attributes below are shadowed by instance attributes, and are 

539 # present just to hold the docstrings for those instance attributes. 

540 

541 pixelization: Pixelization 

542 """Pixelization instance that can compute regions from IDs and IDs from 

543 points (`sphgeom.Pixelization`). 

544 """