Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"] 

25 

26from typing import ( 

27 Any, 

28 AbstractSet, 

29 Dict, 

30 Iterable, 

31 Optional, 

32 Set, 

33 Type, 

34 TYPE_CHECKING, 

35) 

36 

37from sqlalchemy import Integer 

38 

39from lsst.sphgeom import Pixelization 

40from ..utils import immutable 

41from ..named import NamedValueSet 

42from .. import ddl 

43 

44if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from .universe import DimensionUniverse 

46 from .graph import DimensionGraph 

47 from .records import DimensionRecord 

48 

49 

50class RelatedDimensions: 

51 """A semi-internal struct containing the names of the dimension elements 

52 related to the one holding the instance of this struct. 

53 

54 This object is used as the type of `DimensionElement._related`, which is 

55 considered private _to the dimensions subpackage_, rather that private or 

56 protected within `DimensionElement` itself. 

57 

58 Parameters 

59 ---------- 

60 required : `set` [ `str` ] 

61 The names of other dimensions that are used to form the (compound) 

62 primary key for this element, as well as foreign keys. 

63 implied : `set` [ `str` ] 

64 The names of other dimensions that are used to define foreign keys 

65 for this element, but not primary keys. 

66 spatial : `str`, optional 

67 The name of a `DimensionElement` whose spatial regions this element's 

68 region aggregates, or the name of this element if it has a region 

69 that is not an aggregate. `None` (default) if this element is not 

70 associated with region. 

71 temporal : `str`, optional 

72 The name of a `DimensionElement` whose timespans this element's 

73 timespan aggregates, or the name of this element if it has a timespan 

74 that is not an aggregate. `None` (default) if this element is not 

75 associated with timespan. 

76 """ 

77 def __init__(self, required: Set[str], implied: Set[str], 

78 spatial: Optional[str] = None, temporal: Optional[str] = None): 

79 self.required = required 

80 self.implied = implied 

81 self.spatial = spatial 

82 self.temporal = temporal 

83 self.dependencies = set(self.required | self.implied) 

84 

85 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies") 

86 

87 def expand(self, universe: DimensionUniverse) -> None: 

88 """Expand ``required`` and ``dependencies`` recursively. 

89 

90 Parameters 

91 ---------- 

92 universe : `DimensionUniverse` 

93 Object containing all other dimension elements. 

94 """ 

95 for req in tuple(self.required): 

96 other = universe.elements[req]._related 

97 self.required.update(other.required) 

98 self.dependencies.update(other.dependencies) 

99 for dep in self.implied: 

100 other = universe.elements[dep]._related 

101 self.dependencies.update(other.dependencies) 

102 

103 required: Set[str] 

104 """The names of dimensions that are used to form the (compound) primary key 

105 for this element, as well as foreign keys. 

106 

107 For true `Dimension` instances, this should be constructed without the 

108 dimension's own name, with the `Dimension` itself adding it later (after 

109 `dependencies` is defined). 

110 """ 

111 

112 implied: Set[str] 

113 """The names of other dimensions that are used to define foreign keys for 

114 this element, but not primary keys. 

115 """ 

116 

117 dependencies: Set[str] 

118 """The names of all dimensions in `required` or `implied`. 

119 

120 Immediately after construction, this is equal to the union of `required` 

121 `implied`. After `expand` is called, this may not be true, as this will 

122 include the required and implied dependencies (recursively) of implied 

123 dependencies, while `implied` is not expanded recursively. 

124 

125 For true `Dimension` instances, this never includes the dimension's own 

126 name. 

127 """ 

128 

129 spatial: Optional[str] 

130 """The name of a dimension element to which this element delegates spatial 

131 region handling, if any (see `DimensionElement.spatial`). 

132 """ 

133 

134 temporal: Optional[str] 

135 """The name of a dimension element to which this element delegates 

136 timespan handling, if any (see `DimensionElement.temporal`). 

137 """ 

138 

139 

140@immutable 

141class DimensionElement: 

142 """A named data-organization concept that defines a label and/or metadata 

143 in the dimensions system. 

144 

145 A `DimensionElement` instance typically corresponds to a table in the 

146 `Registry`; the rows in that table are represented by instances of a 

147 `DimensionRecord` subclass. Most `DimensionElement` instances are 

148 instances of its `Dimension` subclass, which is used for elements that can 

149 be used as data ID keys. The base class itself can be used for other 

150 dimension tables that provide metadata keyed by true dimensions. 

151 

152 Parameters 

153 ---------- 

154 name : `str` 

155 Name of the element. Used as at least part of the table name, if 

156 the dimension in associated with a database table. 

157 related : `RelatedDimensions` 

158 Struct containing the names of related dimensions. 

159 metadata : iterable of `FieldSpec` 

160 Additional metadata fields included in this element's table. 

161 cached : `bool` 

162 Whether `Registry` should cache records of this element in-memory. 

163 viewOf : `str`, optional 

164 Name of another table this element's records should be drawn from. The 

165 fields of this table must be a superset of the fields of the element. 

166 

167 Notes 

168 ----- 

169 `DimensionElement` instances should always be constructed by and retreived 

170 from a `DimensionUniverse`. They are immutable after they are fully 

171 constructed, and should never be copied. 

172 

173 Pickling a `DimensionElement` just records its name and universe; 

174 unpickling one actually just looks up the element via the singleton 

175 dictionary of all universes. This allows pickle to be used to transfer 

176 elements between processes, but only when each process initializes its own 

177 instance of the same `DimensionUniverse`. 

178 """ 

179 

180 def __init__(self, name: str, *, 

181 related: RelatedDimensions, 

182 metadata: Iterable[ddl.FieldSpec] = (), 

183 cached: bool = False, 

184 viewOf: Optional[str] = None, 

185 alwaysJoin: bool = False): 

186 self.name = name 

187 self._related = related 

188 self.metadata = NamedValueSet(metadata) 

189 self.metadata.freeze() 

190 self.cached = cached 

191 self.viewOf = viewOf 

192 self.alwaysJoin = alwaysJoin 

193 

194 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]) -> None: 

195 """Finish construction of the element and add it to the given universe. 

196 

197 For internal use by `DimensionUniverse` only. 

198 

199 Parameters 

200 ---------- 

201 universe : `DimensionUniverse` 

202 The under-construction dimension universe. It can be relied upon 

203 to contain all dimensions that this element requires or implies. 

204 elementsToDo : `dict` [ `str`, `DimensionElement` ] 

205 A dictionary containing all dimension elements that have not yet 

206 been added to ``universe``, keyed by name. 

207 """ 

208 # Attach set self.universe and add self to universe attributes; 

209 # let subclasses override which attributes by calling a separate 

210 # method. 

211 self._attachToUniverse(universe) 

212 # Expand dependencies. 

213 self._related.expand(universe) 

214 # Define public DimensionElement versions of some of the name sets 

215 # in self._related. 

216 self.required = NamedValueSet(universe.sorted(self._related.required)) 

217 self.required.freeze() 

218 self.implied = NamedValueSet(universe.sorted(self._related.implied)) 

219 self.implied.freeze() 

220 # Set self.spatial and self.temporal to DimensionElement instances from 

221 # the private _related versions of those. 

222 for s in ("spatial", "temporal"): 

223 targetName = getattr(self._related, s, None) 

224 if targetName is None: 

225 target = None 

226 elif targetName == self.name: 

227 target = self 

228 else: 

229 target = universe.elements.get(targetName) 

230 if target is None: 

231 try: 

232 target = elementsToDo[targetName] 

233 except KeyError as err: 

234 raise LookupError( 

235 f"Could not find {s} provider {targetName} for {self.name}." 

236 ) from err 

237 setattr(self, s, target) 

238 # Attach a DimensionGraph that provides the public API for getting 

239 # at requirements. Again delegate to subclasses. 

240 self._attachGraph() 

241 # Create and attach a DimensionRecord subclass to hold values of this 

242 # dimension type. 

243 from .records import _subclassDimensionRecord 

244 self.RecordClass = _subclassDimensionRecord(self) 

245 

246 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

247 """Add the element to the given universe. 

248 

249 Called only by `_finish`, but may be overridden by subclasses. 

250 """ 

251 self.universe = universe 

252 self.universe.elements.add(self) 

253 

254 def _attachGraph(self) -> None: 

255 """Initialize the `graph` attribute for this element. 

256 

257 Called only by `_finish`, but may be overridden by subclasses. 

258 """ 

259 from .graph import DimensionGraph 

260 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False) 

261 

262 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

263 """Return `True` if this element should be included in `DimensionGraph` 

264 that includes the named dimensions. 

265 

266 For internal use by `DimensionGraph` only. 

267 """ 

268 return self._related.required.issubset(dimensionNames) 

269 

270 def __str__(self) -> str: 

271 return self.name 

272 

273 def __repr__(self) -> str: 

274 return f"{type(self).__name__}({self.name})" 

275 

276 def __eq__(self, other: Any) -> bool: 

277 try: 

278 return self.name == other.name 

279 except AttributeError: 

280 return self.name == other 

281 

282 def __hash__(self) -> int: 

283 return hash(self.name) 

284 

285 def __lt__(self, other: DimensionElement) -> bool: 

286 try: 

287 return self.universe._elementIndices[self] < self.universe._elementIndices[other] 

288 except KeyError: 

289 return NotImplemented 

290 

291 def __le__(self, other: DimensionElement) -> bool: 

292 try: 

293 return self.universe._elementIndices[self] <= self.universe._elementIndices[other] 

294 except KeyError: 

295 return NotImplemented 

296 

297 def __gt__(self, other: DimensionElement) -> bool: 

298 try: 

299 return self.universe._elementIndices[self] > self.universe._elementIndices[other] 

300 except KeyError: 

301 return NotImplemented 

302 

303 def __ge__(self, other: DimensionElement) -> bool: 

304 try: 

305 return self.universe._elementIndices[self] >= self.universe._elementIndices[other] 

306 except KeyError: 

307 return NotImplemented 

308 

309 def hasTable(self) -> bool: 

310 """Return `True` if this element is associated with a table 

311 (even if that table "belongs" to another element). 

312 

313 Instances of the `DimensionElement` base class itself are always 

314 associated with tables. 

315 """ 

316 return True 

317 

318 @classmethod 

319 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

320 """Callable used for unpickling. 

321 

322 For internal use only. 

323 """ 

324 return universe.elements[name] 

325 

326 def __reduce__(self) -> tuple: 

327 return (self._unpickle, (self.universe, self.name)) 

328 

329 # Class attributes below are shadowed by instance attributes, and are 

330 # present just to hold the docstrings for those instance attributes. 

331 

332 universe: DimensionUniverse 

333 """The universe of all compatible dimensions with which this element is 

334 associated (`DimensionUniverse`). 

335 """ 

336 

337 name: str 

338 """Unique name for this dimension element (`str`). 

339 """ 

340 

341 graph: DimensionGraph 

342 """Minimal graph that includes this element (`DimensionGraph`). 

343 

344 ``self.graph.required`` includes all dimensions whose primary key values 

345 are sufficient (often necessary) to uniquely identify ``self`` 

346 (including ``self` if ``isinstance(self, Dimension)``. 

347 ``self.graph.implied`` includes all dimensions also identified (possibly 

348 recursively) by this set. 

349 """ 

350 

351 required: NamedValueSet[Dimension] 

352 """Dimensions that are sufficient (often necessary) to uniquely identify 

353 a record of this dimension element. 

354 

355 For elements with a database representation, these dimension are exactly 

356 those used to form the (possibly compound) primary key, and all dimensions 

357 here that are not ``self`` are also used to form foreign keys. 

358 

359 For `Dimension` instances, this should be exactly the same as 

360 ``graph.required``, but that may not be true for `DimensionElement` 

361 instances in general. When they differ, there are multiple combinations 

362 of dimensions that uniquely identify this element, but this one is more 

363 direct. 

364 """ 

365 

366 implied: NamedValueSet[Dimension] 

367 """Other dimensions that are uniquely identified directly by a record of 

368 this dimension element. 

369 

370 For elements with a database representation, these are exactly the 

371 dimensions used to form foreign key constraints whose fields are not 

372 (wholly) also part of the primary key. 

373 

374 Unlike ``self.graph.implied``, this set is not expanded recursively. 

375 """ 

376 

377 spatial: Optional[DimensionElement] 

378 """A `DimensionElement` whose spatial regions this element's region 

379 aggregates, or ``self`` if it has a region that is not an aggregate 

380 (`DimensionElement` or `None`). 

381 """ 

382 

383 temporal: Optional[DimensionElement] 

384 """A `DimensionElement` whose timespans this element's timespan 

385 aggregates, or ``self`` if it has a timespan that is not an aggregate 

386 (`DimensionElement` or `None`). 

387 """ 

388 

389 metadata: NamedValueSet[ddl.FieldSpec] 

390 """Additional metadata fields included in this element's table 

391 (`NamedValueSet` of `FieldSpec`). 

392 """ 

393 

394 RecordClass: Type[DimensionRecord] 

395 """The `DimensionRecord` subclass used to hold records for this element 

396 (`type`). 

397 

398 Because `DimensionRecord` subclasses are generated dynamically, this type 

399 cannot be imported directly and hence canonly be obtained from this 

400 attribute. 

401 """ 

402 

403 cached: bool 

404 """Whether `Registry` should cache records of this element in-memory 

405 (`bool`). 

406 """ 

407 

408 viewOf: Optional[str] 

409 """Name of another table this elements records are drawn from (`str` or 

410 `None`). 

411 """ 

412 

413 

414@immutable 

415class Dimension(DimensionElement): 

416 """A named data-organization concept that can be used as a key in a data 

417 ID. 

418 

419 Parameters 

420 ---------- 

421 name : `str` 

422 Name of the dimension. Used as at least part of the table name, if 

423 the dimension in associated with a database table, and as an alternate 

424 key (instead of the instance itself) in data IDs. 

425 related : `RelatedDimensions` 

426 Struct containing the names of related dimensions. 

427 uniqueKeys : iterable of `FieldSpec` 

428 Fields that can *each* be used to uniquely identify this dimension 

429 (once all required dependencies are also identified). The first entry 

430 will be used as the table's primary key and as a foriegn key field in 

431 the fields of dependent tables. 

432 kwds 

433 Additional keyword arguments are forwarded to the `DimensionElement` 

434 constructor. 

435 """ 

436 

437 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec], 

438 **kwargs: Any): 

439 related.required.add(name) 

440 super().__init__(name, related=related, **kwargs) 

441 self.uniqueKeys = NamedValueSet(uniqueKeys) 

442 self.uniqueKeys.freeze() 

443 self.primaryKey, *alternateKeys = uniqueKeys 

444 self.alternateKeys = NamedValueSet(alternateKeys) 

445 self.alternateKeys.freeze() 

446 

447 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

448 # Docstring inherited from DimensionElement._attachToUniverse. 

449 super()._attachToUniverse(universe) 

450 universe.dimensions.add(self) 

451 

452 def _attachGraph(self) -> None: 

453 # Docstring inherited from DimensionElement._attachGraph. 

454 from .graph import DimensionGraph 

455 self.graph = DimensionGraph(self.universe, 

456 names=self._related.dependencies.union([self.name]), 

457 conform=False) 

458 

459 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

460 # Docstring inherited from DimensionElement._shouldBeInGraph. 

461 return self.name in dimensionNames 

462 

463 # Class attributes below are shadowed by instance attributes, and are 

464 # present just to hold the docstrings for those instance attributes. 

465 

466 uniqueKeys: NamedValueSet[ddl.FieldSpec] 

467 """All fields that can individually be used to identify records of this 

468 element, given the primary keys of all required dependencies 

469 (`NamedValueSet` of `FieldSpec`). 

470 """ 

471 

472 primaryKey: ddl.FieldSpec 

473 """The primary key field for this dimension (`FieldSpec`). 

474 

475 Note that the database primary keys for dimension tables are in general 

476 compound; this field is the only field in the database primary key that is 

477 not also a foreign key (to a required dependency dimension table). 

478 """ 

479 

480 alternateKeys: NamedValueSet[ddl.FieldSpec] 

481 """Additional unique key fields for this dimension that are not the the 

482 primary key (`NamedValueSet` of `FieldSpec`). 

483 """ 

484 

485 

486@immutable 

487class SkyPixDimension(Dimension): 

488 """A special `Dimension` subclass for hierarchical pixelizations of the 

489 sky. 

490 

491 Unlike most other dimensions, skypix dimension records are not stored in 

492 the database, as these records only contain an integer pixel ID and a 

493 region on the sky, and each of these can be computed directly from the 

494 other. 

495 

496 Parameters 

497 ---------- 

498 name : `str` 

499 Name of the dimension. By convention, this is a lowercase string 

500 abbreviation for the pixelization followed by its integer level, 

501 such as "htm7". 

502 pixelization : `sphgeom.Pixelization` 

503 Pixelization instance that can compute regions from IDs and IDs from 

504 points. 

505 """ 

506 

507 def __init__(self, name: str, pixelization: Pixelization): 

508 related = RelatedDimensions(required=set(), implied=set(), spatial=name) 

509 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)] 

510 super().__init__(name, related=related, uniqueKeys=uniqueKeys) 

511 self.pixelization = pixelization 

512 

513 def hasTable(self) -> bool: 

514 # Docstring inherited from DimensionElement.hasTable. 

515 return False 

516 

517 # Class attributes below are shadowed by instance attributes, and are 

518 # present just to hold the docstrings for those instance attributes. 

519 

520 pixelization: Pixelization 

521 """Pixelization instance that can compute regions from IDs and IDs from 

522 points (`sphgeom.Pixelization`). 

523 """