Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"] 

25 

26from typing import ( 

27 AbstractSet, 

28 Dict, 

29 Iterable, 

30 Optional, 

31 Set, 

32 TYPE_CHECKING, 

33) 

34 

35from sqlalchemy import Integer 

36 

37from lsst.sphgeom import Pixelization 

38from ..utils import NamedValueSet, immutable 

39from .. import ddl 

40from .records import _subclassDimensionRecord 

41from .graph import DimensionGraph 

42 

43if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .universe import DimensionUniverse 

45 

46 

47class RelatedDimensions: 

48 """A semi-internal struct containing the names of the dimension elements 

49 related to the one holding the instance of this struct. 

50 

51 This object is used as the type of `DimensionElement._related`, which is 

52 considered private _to the dimensions subpackage_, rather that private or 

53 protected within `DimensionElement` itself. 

54 

55 Parameters 

56 ---------- 

57 required : `set` [ `str` ] 

58 The names of other dimensions that are used to form the (compound) 

59 primary key for this element, as well as foreign keys. 

60 implied : `set` [ `str` ] 

61 The names of other dimensions that are used to define foreign keys 

62 for this element, but not primary keys. 

63 spatial : `str`, optional 

64 The name of a `DimensionElement` whose spatial regions this element's 

65 region aggregates, or the name of this element if it has a region 

66 that is not an aggregate. `None` (default) if this element is not 

67 associated with region. 

68 temporal : `str`, optional 

69 The name of a `DimensionElement` whose timespans this element's 

70 timespan aggregates, or the name of this element if it has a timespan 

71 that is not an aggregate. `None` (default) if this element is not 

72 associated with timespan. 

73 """ 

74 def __init__(self, required: Set[str], implied: Set[str], 

75 spatial: Optional[str] = None, temporal: Optional[str] = None): 

76 self.required = required 

77 self.implied = implied 

78 self.spatial = spatial 

79 self.temporal = temporal 

80 self.dependencies = set(self.required | self.implied) 

81 

82 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies") 

83 

84 def expand(self, universe: DimensionUniverse): 

85 """Expand ``required`` and ``dependencies`` recursively. 

86 

87 Parameters 

88 ---------- 

89 universe : `DimensionUniverse` 

90 Object containing all other dimension elements. 

91 """ 

92 for req in tuple(self.required): 

93 other = universe.elements[req]._related 

94 self.required.update(other.required) 

95 self.dependencies.update(other.dependencies) 

96 for dep in self.implied: 

97 other = universe.elements[dep]._related 

98 self.dependencies.update(other.dependencies) 

99 

100 required: Set[str] 

101 """The names of dimensions that are used to form the (compound) primary key 

102 for this element, as well as foreign keys. 

103 

104 For true `Dimension` instances, this should be constructed without the 

105 dimension's own name, with the `Dimension` itself adding it later (after 

106 `dependencies` is defined). 

107 """ 

108 

109 implied: Set[str] 

110 """The names of other dimensions that are used to define foreign keys for 

111 this element, but not primary keys. 

112 """ 

113 

114 dependencies: Set[str] 

115 """The names of all dimensions in `required` or `implied`. 

116 

117 Immediately after construction, this is equal to the union of `required` 

118 `implied`. After `expand` is called, this may not be true, as this will 

119 include the required and implied dependencies (recursively) of implied 

120 dependencies, while `implied` is not expanded recursively. 

121 

122 For true `Dimension` instances, this never includes the dimension's own 

123 name. 

124 """ 

125 

126 spatial: Optional[str] 

127 """The name of a dimension element to which this element delegates spatial 

128 region handling, if any (see `DimensionElement.spatial`). 

129 """ 

130 

131 temporal: Optional[str] 

132 """The name of a dimension element to which this element delegates 

133 timespan handling, if any (see `DimensionElement.temporal`). 

134 """ 

135 

136 

137@immutable 

138class DimensionElement: 

139 """A named data-organization concept that defines a label and/or metadata 

140 in the dimensions system. 

141 

142 A `DimensionElement` instance typically corresponds to a table in the 

143 `Registry`; the rows in that table are represented by instances of a 

144 `DimensionRecord` subclass. Most `DimensionElement` instances are 

145 instances of its `Dimension` subclass, which is used for elements that can 

146 be used as data ID keys. The base class itself can be used for other 

147 dimension tables that provide metadata keyed by true dimensions. 

148 

149 Parameters 

150 ---------- 

151 name : `str` 

152 Name of the element. Used as at least part of the table name, if 

153 the dimension in associated with a database table. 

154 related : `RelatedDimensions` 

155 Struct containing the names of related dimensions. 

156 metadata : iterable of `FieldSpec` 

157 Additional metadata fields included in this element's table. 

158 cached : `bool` 

159 Whether `Registry` should cache records of this element in-memory. 

160 viewOf : `str`, optional 

161 Name of another table this element's records should be drawn from. The 

162 fields of this table must be a superset of the fields of the element. 

163 

164 Notes 

165 ----- 

166 `DimensionElement` instances should always be constructed by and retreived 

167 from a `DimensionUniverse`. They are immutable after they are fully 

168 constructed, and should never be copied. 

169 

170 Pickling a `DimensionElement` just records its name and universe; 

171 unpickling one actually just looks up the element via the singleton 

172 dictionary of all universes. This allows pickle to be used to transfer 

173 elements between processes, but only when each process initializes its own 

174 instance of the same `DimensionUniverse`. 

175 """ 

176 

177 def __init__(self, name: str, *, 

178 related: RelatedDimensions, 

179 metadata: Iterable[ddl.FieldSpec] = (), 

180 cached: bool = False, 

181 viewOf: Optional[str] = None, 

182 alwaysJoin: bool = False): 

183 self.name = name 

184 self._related = related 

185 self.metadata = NamedValueSet(metadata) 

186 self.metadata.freeze() 

187 self.cached = cached 

188 self.viewOf = viewOf 

189 self.alwaysJoin = alwaysJoin 

190 

191 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]): 

192 """Finish construction of the element and add it to the given universe. 

193 

194 For internal use by `DimensionUniverse` only. 

195 

196 Parameters 

197 ---------- 

198 universe : `DimensionUniverse` 

199 The under-construction dimension universe. It can be relied upon 

200 to contain all dimensions that this element requires or implies. 

201 elementsToDo : `dict` [ `str`, `DimensionElement` ] 

202 A dictionary containing all dimension elements that have not yet 

203 been added to ``universe``, keyed by name. 

204 """ 

205 # Attach set self.universe and add self to universe attributes; 

206 # let subclasses override which attributes by calling a separate 

207 # method. 

208 self._attachToUniverse(universe) 

209 # Expand dependencies. 

210 self._related.expand(universe) 

211 # Define public DimensionElement versions of some of the name sets 

212 # in self._related. 

213 self.required = NamedValueSet(universe.sorted(self._related.required)) 

214 self.required.freeze() 

215 self.implied = NamedValueSet(universe.sorted(self._related.implied)) 

216 self.implied.freeze() 

217 # Set self.spatial and self.temporal to DimensionElement instances from 

218 # the private _related versions of those. 

219 for s in ("spatial", "temporal"): 

220 targetName = getattr(self._related, s, None) 

221 if targetName is None: 

222 target = None 

223 elif targetName == self.name: 

224 target = self 

225 else: 

226 target = universe.elements.get(targetName) 

227 if target is None: 

228 try: 

229 target = elementsToDo[targetName] 

230 except KeyError as err: 

231 raise LookupError( 

232 f"Could not find {s} provider {targetName} for {self.name}." 

233 ) from err 

234 setattr(self, s, target) 

235 # Attach a DimensionGraph that provides the public API for getting 

236 # at requirements. Again delegate to subclasses. 

237 self._attachGraph() 

238 # Create and attach a DimensionRecord subclass to hold values of this 

239 # dimension type. 

240 self.RecordClass = _subclassDimensionRecord(self) 

241 

242 def _attachToUniverse(self, universe: DimensionUniverse): 

243 """Add the element to the given universe. 

244 

245 Called only by `_finish`, but may be overridden by subclasses. 

246 """ 

247 self.universe = universe 

248 self.universe.elements.add(self) 

249 

250 def _attachGraph(self): 

251 """Initialize the `graph` attribute for this element. 

252 

253 Called only by `_finish`, but may be overridden by subclasses. 

254 """ 

255 from .graph import DimensionGraph 

256 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False) 

257 

258 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]): 

259 """Return `True` if this element should be included in `DimensionGraph` 

260 that includes the named dimensions. 

261 

262 For internal use by `DimensionGraph` only. 

263 """ 

264 return self._related.required.issubset(dimensionNames) 

265 

266 def __str__(self) -> str: 

267 return self.name 

268 

269 def __repr__(self) -> str: 

270 return f"{type(self).__name__}({self.name})" 

271 

272 def __eq__(self, other) -> bool: 

273 try: 

274 return self.name == other.name 

275 except AttributeError: 

276 return self.name == other 

277 

278 def __hash__(self) -> int: 

279 return hash(self.name) 

280 

281 def __lt__(self, other) -> bool: 

282 try: 

283 return self.universe._elementIndices[self] < self.universe._elementIndices[other] 

284 except KeyError: 

285 return NotImplemented 

286 

287 def __le__(self, other) -> bool: 

288 try: 

289 return self.universe._elementIndices[self] <= self.universe._elementIndices[other] 

290 except KeyError: 

291 return NotImplemented 

292 

293 def __gt__(self, other) -> bool: 

294 try: 

295 return self.universe._elementIndices[self] > self.universe._elementIndices[other] 

296 except KeyError: 

297 return NotImplemented 

298 

299 def __ge__(self, other) -> bool: 

300 try: 

301 return self.universe._elementIndices[self] >= self.universe._elementIndices[other] 

302 except KeyError: 

303 return NotImplemented 

304 

305 def hasTable(self) -> bool: 

306 """Return `True` if this element is associated with a table 

307 (even if that table "belongs" to another element). 

308 

309 Instances of the `DimensionElement` base class itself are always 

310 associated with tables. 

311 """ 

312 return True 

313 

314 @classmethod 

315 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

316 """Callable used for unpickling. 

317 

318 For internal use only. 

319 """ 

320 return universe.elements[name] 

321 

322 def __reduce__(self) -> tuple: 

323 return (self._unpickle, (self.universe, self.name)) 

324 

325 # Class attributes below are shadowed by instance attributes, and are 

326 # present just to hold the docstrings for those instance attributes. 

327 

328 universe: DimensionUniverse 

329 """The universe of all compatible dimensions with which this element is 

330 associated (`DimensionUniverse`). 

331 """ 

332 

333 name: str 

334 """Unique name for this dimension element (`str`). 

335 """ 

336 

337 graph: DimensionGraph 

338 """Minimal graph that includes this element (`DimensionGraph`). 

339 

340 ``self.graph.required`` includes all dimensions whose primary key values 

341 are sufficient (often necessary) to uniquely identify ``self`` 

342 (including ``self` if ``isinstance(self, Dimension)``. 

343 ``self.graph.implied`` includes all dimensions also identified (possibly 

344 recursively) by this set. 

345 """ 

346 

347 required: NamedValueSet[Dimension] 

348 """Dimensions that are sufficient (often necessary) to uniquely identify 

349 a record of this dimension element. 

350 

351 For elements with a database representation, these dimension are exactly 

352 those used to form the (possibly compound) primary key, and all dimensions 

353 here that are not ``self`` are also used to form foreign keys. 

354 

355 For `Dimension` instances, this should be exactly the same as 

356 ``graph.required``, but that may not be true for `DimensionElement` 

357 instances in general. When they differ, there are multiple combinations 

358 of dimensions that uniquely identify this element, but this one is more 

359 direct. 

360 """ 

361 

362 implied: NamedValueSet[Dimension] 

363 """Other dimensions that are uniquely identified directly by a record of 

364 this dimension element. 

365 

366 For elements with a database representation, these are exactly the 

367 dimensions used to form foreign key constraints whose fields are not 

368 (wholly) also part of the primary key. 

369 

370 Unlike ``self.graph.implied``, this set is not expanded recursively. 

371 """ 

372 

373 spatial: Optional[DimensionElement] 

374 """A `DimensionElement` whose spatial regions this element's region 

375 aggregates, or ``self`` if it has a region that is not an aggregate 

376 (`DimensionElement` or `None`). 

377 """ 

378 

379 temporal: Optional[DimensionElement] 

380 """A `DimensionElement` whose timespans this element's timespan 

381 aggregates, or ``self`` if it has a timespan that is not an aggregate 

382 (`DimensionElement` or `None`). 

383 """ 

384 

385 metadata: NamedValueSet[ddl.FieldSpec] 

386 """Additional metadata fields included in this element's table 

387 (`NamedValueSet` of `FieldSpec`). 

388 """ 

389 

390 RecordClass: type 

391 """The `DimensionRecord` subclass used to hold records for this element 

392 (`type`). 

393 

394 Because `DimensionRecord` subclasses are generated dynamically, this type 

395 cannot be imported directly and hence canonly be obtained from this 

396 attribute. 

397 """ 

398 

399 cached: bool 

400 """Whether `Registry` should cache records of this element in-memory 

401 (`bool`). 

402 """ 

403 

404 viewOf: Optional[str] 

405 """Name of another table this elements records are drawn from (`str` or 

406 `None`). 

407 """ 

408 

409 

410@immutable 

411class Dimension(DimensionElement): 

412 """A named data-organization concept that can be used as a key in a data 

413 ID. 

414 

415 Parameters 

416 ---------- 

417 name : `str` 

418 Name of the dimension. Used as at least part of the table name, if 

419 the dimension in associated with a database table, and as an alternate 

420 key (instead of the instance itself) in data IDs. 

421 related : `RelatedDimensions` 

422 Struct containing the names of related dimensions. 

423 uniqueKeys : iterable of `FieldSpec` 

424 Fields that can *each* be used to uniquely identify this dimension 

425 (once all required dependencies are also identified). The first entry 

426 will be used as the table's primary key and as a foriegn key field in 

427 the fields of dependent tables. 

428 kwds 

429 Additional keyword arguments are forwarded to the `DimensionElement` 

430 constructor. 

431 """ 

432 

433 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec], **kwds): 

434 related.required.add(name) 

435 super().__init__(name, related=related, **kwds) 

436 self.uniqueKeys = NamedValueSet(uniqueKeys) 

437 self.uniqueKeys.freeze() 

438 self.primaryKey, *alternateKeys = uniqueKeys 

439 self.alternateKeys = NamedValueSet(alternateKeys) 

440 self.alternateKeys.freeze() 

441 

442 def _attachToUniverse(self, universe: DimensionUniverse): 

443 # Docstring inherited from DimensionElement._attachToUniverse. 

444 super()._attachToUniverse(universe) 

445 universe.dimensions.add(self) 

446 

447 def _attachGraph(self): 

448 # Docstring inherited from DimensionElement._attachGraph. 

449 self.graph = DimensionGraph(self.universe, 

450 names=self._related.dependencies.union([self.name]), 

451 conform=False) 

452 

453 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]): 

454 # Docstring inherited from DimensionElement._shouldBeInGraph. 

455 return self.name in dimensionNames 

456 

457 # Class attributes below are shadowed by instance attributes, and are 

458 # present just to hold the docstrings for those instance attributes. 

459 

460 uniqueKeys: NamedValueSet[ddl.FieldSpec] 

461 """All fields that can individually be used to identify records of this 

462 element, given the primary keys of all required dependencies 

463 (`NamedValueSet` of `FieldSpec`). 

464 """ 

465 

466 primaryKey: ddl.FieldSpec 

467 """The primary key field for this dimension (`FieldSpec`). 

468 

469 Note that the database primary keys for dimension tables are in general 

470 compound; this field is the only field in the database primary key that is 

471 not also a foreign key (to a required dependency dimension table). 

472 """ 

473 

474 alternateKeys: NamedValueSet[ddl.FieldSpec] 

475 """Additional unique key fields for this dimension that are not the the 

476 primary key (`NamedValueSet` of `FieldSpec`). 

477 """ 

478 

479 

480@immutable 

481class SkyPixDimension(Dimension): 

482 """A special `Dimension` subclass for hierarchical pixelizations of the 

483 sky. 

484 

485 Unlike most other dimensions, skypix dimension records are not stored in 

486 the database, as these records only contain an integer pixel ID and a 

487 region on the sky, and each of these can be computed directly from the 

488 other. 

489 

490 Parameters 

491 ---------- 

492 name : `str` 

493 Name of the dimension. By convention, this is a lowercase string 

494 abbreviation for the pixelization followed by its integer level, 

495 such as "htm7". 

496 pixelization : `sphgeom.Pixelization` 

497 Pixelization instance that can compute regions from IDs and IDs from 

498 points. 

499 """ 

500 

501 def __init__(self, name: str, pixelization: Pixelization): 

502 related = RelatedDimensions(required=set(), implied=set(), spatial=self) 

503 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)] 

504 super().__init__(name, related=related, uniqueKeys=uniqueKeys) 

505 self.pixelization = pixelization 

506 

507 def hasTable(self) -> bool: 

508 # Docstring inherited from DimensionElement.hasTable. 

509 return False 

510 

511 # Class attributes below are shadowed by instance attributes, and are 

512 # present just to hold the docstrings for those instance attributes. 

513 

514 pixelization: Pixelization 

515 """Pixelization instance that can compute regions from IDs and IDs from 

516 points (`sphgeom.Pixelization`). 

517 """