Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"] 

25 

26from typing import ( 

27 Any, 

28 AbstractSet, 

29 Dict, 

30 Iterable, 

31 Optional, 

32 Set, 

33 TYPE_CHECKING, 

34) 

35 

36from sqlalchemy import Integer 

37 

38from lsst.sphgeom import Pixelization 

39from ..utils import immutable 

40from ..named import NamedValueSet 

41from .. import ddl 

42 

43if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .universe import DimensionUniverse 

45 from .graph import DimensionGraph 

46 

47 

48class RelatedDimensions: 

49 """A semi-internal struct containing the names of the dimension elements 

50 related to the one holding the instance of this struct. 

51 

52 This object is used as the type of `DimensionElement._related`, which is 

53 considered private _to the dimensions subpackage_, rather that private or 

54 protected within `DimensionElement` itself. 

55 

56 Parameters 

57 ---------- 

58 required : `set` [ `str` ] 

59 The names of other dimensions that are used to form the (compound) 

60 primary key for this element, as well as foreign keys. 

61 implied : `set` [ `str` ] 

62 The names of other dimensions that are used to define foreign keys 

63 for this element, but not primary keys. 

64 spatial : `str`, optional 

65 The name of a `DimensionElement` whose spatial regions this element's 

66 region aggregates, or the name of this element if it has a region 

67 that is not an aggregate. `None` (default) if this element is not 

68 associated with region. 

69 temporal : `str`, optional 

70 The name of a `DimensionElement` whose timespans this element's 

71 timespan aggregates, or the name of this element if it has a timespan 

72 that is not an aggregate. `None` (default) if this element is not 

73 associated with timespan. 

74 """ 

75 def __init__(self, required: Set[str], implied: Set[str], 

76 spatial: Optional[str] = None, temporal: Optional[str] = None): 

77 self.required = required 

78 self.implied = implied 

79 self.spatial = spatial 

80 self.temporal = temporal 

81 self.dependencies = set(self.required | self.implied) 

82 

83 __slots__ = ("required", "implied", "spatial", "temporal", "dependencies") 

84 

85 def expand(self, universe: DimensionUniverse) -> None: 

86 """Expand ``required`` and ``dependencies`` recursively. 

87 

88 Parameters 

89 ---------- 

90 universe : `DimensionUniverse` 

91 Object containing all other dimension elements. 

92 """ 

93 for req in tuple(self.required): 

94 other = universe.elements[req]._related 

95 self.required.update(other.required) 

96 self.dependencies.update(other.dependencies) 

97 for dep in self.implied: 

98 other = universe.elements[dep]._related 

99 self.dependencies.update(other.dependencies) 

100 

101 required: Set[str] 

102 """The names of dimensions that are used to form the (compound) primary key 

103 for this element, as well as foreign keys. 

104 

105 For true `Dimension` instances, this should be constructed without the 

106 dimension's own name, with the `Dimension` itself adding it later (after 

107 `dependencies` is defined). 

108 """ 

109 

110 implied: Set[str] 

111 """The names of other dimensions that are used to define foreign keys for 

112 this element, but not primary keys. 

113 """ 

114 

115 dependencies: Set[str] 

116 """The names of all dimensions in `required` or `implied`. 

117 

118 Immediately after construction, this is equal to the union of `required` 

119 `implied`. After `expand` is called, this may not be true, as this will 

120 include the required and implied dependencies (recursively) of implied 

121 dependencies, while `implied` is not expanded recursively. 

122 

123 For true `Dimension` instances, this never includes the dimension's own 

124 name. 

125 """ 

126 

127 spatial: Optional[str] 

128 """The name of a dimension element to which this element delegates spatial 

129 region handling, if any (see `DimensionElement.spatial`). 

130 """ 

131 

132 temporal: Optional[str] 

133 """The name of a dimension element to which this element delegates 

134 timespan handling, if any (see `DimensionElement.temporal`). 

135 """ 

136 

137 

138@immutable 

139class DimensionElement: 

140 """A named data-organization concept that defines a label and/or metadata 

141 in the dimensions system. 

142 

143 A `DimensionElement` instance typically corresponds to a table in the 

144 `Registry`; the rows in that table are represented by instances of a 

145 `DimensionRecord` subclass. Most `DimensionElement` instances are 

146 instances of its `Dimension` subclass, which is used for elements that can 

147 be used as data ID keys. The base class itself can be used for other 

148 dimension tables that provide metadata keyed by true dimensions. 

149 

150 Parameters 

151 ---------- 

152 name : `str` 

153 Name of the element. Used as at least part of the table name, if 

154 the dimension in associated with a database table. 

155 related : `RelatedDimensions` 

156 Struct containing the names of related dimensions. 

157 metadata : iterable of `FieldSpec` 

158 Additional metadata fields included in this element's table. 

159 cached : `bool` 

160 Whether `Registry` should cache records of this element in-memory. 

161 viewOf : `str`, optional 

162 Name of another table this element's records should be drawn from. The 

163 fields of this table must be a superset of the fields of the element. 

164 

165 Notes 

166 ----- 

167 `DimensionElement` instances should always be constructed by and retreived 

168 from a `DimensionUniverse`. They are immutable after they are fully 

169 constructed, and should never be copied. 

170 

171 Pickling a `DimensionElement` just records its name and universe; 

172 unpickling one actually just looks up the element via the singleton 

173 dictionary of all universes. This allows pickle to be used to transfer 

174 elements between processes, but only when each process initializes its own 

175 instance of the same `DimensionUniverse`. 

176 """ 

177 

178 def __init__(self, name: str, *, 

179 related: RelatedDimensions, 

180 metadata: Iterable[ddl.FieldSpec] = (), 

181 cached: bool = False, 

182 viewOf: Optional[str] = None, 

183 alwaysJoin: bool = False): 

184 self.name = name 

185 self._related = related 

186 self.metadata = NamedValueSet(metadata) 

187 self.metadata.freeze() 

188 self.cached = cached 

189 self.viewOf = viewOf 

190 self.alwaysJoin = alwaysJoin 

191 

192 def _finish(self, universe: DimensionUniverse, elementsToDo: Dict[str, DimensionElement]) -> None: 

193 """Finish construction of the element and add it to the given universe. 

194 

195 For internal use by `DimensionUniverse` only. 

196 

197 Parameters 

198 ---------- 

199 universe : `DimensionUniverse` 

200 The under-construction dimension universe. It can be relied upon 

201 to contain all dimensions that this element requires or implies. 

202 elementsToDo : `dict` [ `str`, `DimensionElement` ] 

203 A dictionary containing all dimension elements that have not yet 

204 been added to ``universe``, keyed by name. 

205 """ 

206 # Attach set self.universe and add self to universe attributes; 

207 # let subclasses override which attributes by calling a separate 

208 # method. 

209 self._attachToUniverse(universe) 

210 # Expand dependencies. 

211 self._related.expand(universe) 

212 # Define public DimensionElement versions of some of the name sets 

213 # in self._related. 

214 self.required = NamedValueSet(universe.sorted(self._related.required)) 

215 self.required.freeze() 

216 self.implied = NamedValueSet(universe.sorted(self._related.implied)) 

217 self.implied.freeze() 

218 # Set self.spatial and self.temporal to DimensionElement instances from 

219 # the private _related versions of those. 

220 for s in ("spatial", "temporal"): 

221 targetName = getattr(self._related, s, None) 

222 if targetName is None: 

223 target = None 

224 elif targetName == self.name: 

225 target = self 

226 else: 

227 target = universe.elements.get(targetName) 

228 if target is None: 

229 try: 

230 target = elementsToDo[targetName] 

231 except KeyError as err: 

232 raise LookupError( 

233 f"Could not find {s} provider {targetName} for {self.name}." 

234 ) from err 

235 setattr(self, s, target) 

236 # Attach a DimensionGraph that provides the public API for getting 

237 # at requirements. Again delegate to subclasses. 

238 self._attachGraph() 

239 # Create and attach a DimensionRecord subclass to hold values of this 

240 # dimension type. 

241 from .records import _subclassDimensionRecord 

242 self.RecordClass = _subclassDimensionRecord(self) 

243 

244 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

245 """Add the element to the given universe. 

246 

247 Called only by `_finish`, but may be overridden by subclasses. 

248 """ 

249 self.universe = universe 

250 self.universe.elements.add(self) 

251 

252 def _attachGraph(self) -> None: 

253 """Initialize the `graph` attribute for this element. 

254 

255 Called only by `_finish`, but may be overridden by subclasses. 

256 """ 

257 from .graph import DimensionGraph 

258 self.graph = DimensionGraph(self.universe, names=self._related.dependencies, conform=False) 

259 

260 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

261 """Return `True` if this element should be included in `DimensionGraph` 

262 that includes the named dimensions. 

263 

264 For internal use by `DimensionGraph` only. 

265 """ 

266 return self._related.required.issubset(dimensionNames) 

267 

268 def __str__(self) -> str: 

269 return self.name 

270 

271 def __repr__(self) -> str: 

272 return f"{type(self).__name__}({self.name})" 

273 

274 def __eq__(self, other: Any) -> bool: 

275 try: 

276 return self.name == other.name 

277 except AttributeError: 

278 return self.name == other 

279 

280 def __hash__(self) -> int: 

281 return hash(self.name) 

282 

283 def __lt__(self, other: DimensionElement) -> bool: 

284 try: 

285 return self.universe._elementIndices[self] < self.universe._elementIndices[other] 

286 except KeyError: 

287 return NotImplemented 

288 

289 def __le__(self, other: DimensionElement) -> bool: 

290 try: 

291 return self.universe._elementIndices[self] <= self.universe._elementIndices[other] 

292 except KeyError: 

293 return NotImplemented 

294 

295 def __gt__(self, other: DimensionElement) -> bool: 

296 try: 

297 return self.universe._elementIndices[self] > self.universe._elementIndices[other] 

298 except KeyError: 

299 return NotImplemented 

300 

301 def __ge__(self, other: DimensionElement) -> bool: 

302 try: 

303 return self.universe._elementIndices[self] >= self.universe._elementIndices[other] 

304 except KeyError: 

305 return NotImplemented 

306 

307 def hasTable(self) -> bool: 

308 """Return `True` if this element is associated with a table 

309 (even if that table "belongs" to another element). 

310 

311 Instances of the `DimensionElement` base class itself are always 

312 associated with tables. 

313 """ 

314 return True 

315 

316 @classmethod 

317 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

318 """Callable used for unpickling. 

319 

320 For internal use only. 

321 """ 

322 return universe.elements[name] 

323 

324 def __reduce__(self) -> tuple: 

325 return (self._unpickle, (self.universe, self.name)) 

326 

327 # Class attributes below are shadowed by instance attributes, and are 

328 # present just to hold the docstrings for those instance attributes. 

329 

330 universe: DimensionUniverse 

331 """The universe of all compatible dimensions with which this element is 

332 associated (`DimensionUniverse`). 

333 """ 

334 

335 name: str 

336 """Unique name for this dimension element (`str`). 

337 """ 

338 

339 graph: DimensionGraph 

340 """Minimal graph that includes this element (`DimensionGraph`). 

341 

342 ``self.graph.required`` includes all dimensions whose primary key values 

343 are sufficient (often necessary) to uniquely identify ``self`` 

344 (including ``self` if ``isinstance(self, Dimension)``. 

345 ``self.graph.implied`` includes all dimensions also identified (possibly 

346 recursively) by this set. 

347 """ 

348 

349 required: NamedValueSet[Dimension] 

350 """Dimensions that are sufficient (often necessary) to uniquely identify 

351 a record of this dimension element. 

352 

353 For elements with a database representation, these dimension are exactly 

354 those used to form the (possibly compound) primary key, and all dimensions 

355 here that are not ``self`` are also used to form foreign keys. 

356 

357 For `Dimension` instances, this should be exactly the same as 

358 ``graph.required``, but that may not be true for `DimensionElement` 

359 instances in general. When they differ, there are multiple combinations 

360 of dimensions that uniquely identify this element, but this one is more 

361 direct. 

362 """ 

363 

364 implied: NamedValueSet[Dimension] 

365 """Other dimensions that are uniquely identified directly by a record of 

366 this dimension element. 

367 

368 For elements with a database representation, these are exactly the 

369 dimensions used to form foreign key constraints whose fields are not 

370 (wholly) also part of the primary key. 

371 

372 Unlike ``self.graph.implied``, this set is not expanded recursively. 

373 """ 

374 

375 spatial: Optional[DimensionElement] 

376 """A `DimensionElement` whose spatial regions this element's region 

377 aggregates, or ``self`` if it has a region that is not an aggregate 

378 (`DimensionElement` or `None`). 

379 """ 

380 

381 temporal: Optional[DimensionElement] 

382 """A `DimensionElement` whose timespans this element's timespan 

383 aggregates, or ``self`` if it has a timespan that is not an aggregate 

384 (`DimensionElement` or `None`). 

385 """ 

386 

387 metadata: NamedValueSet[ddl.FieldSpec] 

388 """Additional metadata fields included in this element's table 

389 (`NamedValueSet` of `FieldSpec`). 

390 """ 

391 

392 RecordClass: type 

393 """The `DimensionRecord` subclass used to hold records for this element 

394 (`type`). 

395 

396 Because `DimensionRecord` subclasses are generated dynamically, this type 

397 cannot be imported directly and hence canonly be obtained from this 

398 attribute. 

399 """ 

400 

401 cached: bool 

402 """Whether `Registry` should cache records of this element in-memory 

403 (`bool`). 

404 """ 

405 

406 viewOf: Optional[str] 

407 """Name of another table this elements records are drawn from (`str` or 

408 `None`). 

409 """ 

410 

411 

412@immutable 

413class Dimension(DimensionElement): 

414 """A named data-organization concept that can be used as a key in a data 

415 ID. 

416 

417 Parameters 

418 ---------- 

419 name : `str` 

420 Name of the dimension. Used as at least part of the table name, if 

421 the dimension in associated with a database table, and as an alternate 

422 key (instead of the instance itself) in data IDs. 

423 related : `RelatedDimensions` 

424 Struct containing the names of related dimensions. 

425 uniqueKeys : iterable of `FieldSpec` 

426 Fields that can *each* be used to uniquely identify this dimension 

427 (once all required dependencies are also identified). The first entry 

428 will be used as the table's primary key and as a foriegn key field in 

429 the fields of dependent tables. 

430 kwds 

431 Additional keyword arguments are forwarded to the `DimensionElement` 

432 constructor. 

433 """ 

434 

435 def __init__(self, name: str, *, related: RelatedDimensions, uniqueKeys: Iterable[ddl.FieldSpec], 

436 **kwargs: Any): 

437 related.required.add(name) 

438 super().__init__(name, related=related, **kwargs) 

439 self.uniqueKeys = NamedValueSet(uniqueKeys) 

440 self.uniqueKeys.freeze() 

441 self.primaryKey, *alternateKeys = uniqueKeys 

442 self.alternateKeys = NamedValueSet(alternateKeys) 

443 self.alternateKeys.freeze() 

444 

445 def _attachToUniverse(self, universe: DimensionUniverse) -> None: 

446 # Docstring inherited from DimensionElement._attachToUniverse. 

447 super()._attachToUniverse(universe) 

448 universe.dimensions.add(self) 

449 

450 def _attachGraph(self) -> None: 

451 # Docstring inherited from DimensionElement._attachGraph. 

452 from .graph import DimensionGraph 

453 self.graph = DimensionGraph(self.universe, 

454 names=self._related.dependencies.union([self.name]), 

455 conform=False) 

456 

457 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]) -> bool: 

458 # Docstring inherited from DimensionElement._shouldBeInGraph. 

459 return self.name in dimensionNames 

460 

461 # Class attributes below are shadowed by instance attributes, and are 

462 # present just to hold the docstrings for those instance attributes. 

463 

464 uniqueKeys: NamedValueSet[ddl.FieldSpec] 

465 """All fields that can individually be used to identify records of this 

466 element, given the primary keys of all required dependencies 

467 (`NamedValueSet` of `FieldSpec`). 

468 """ 

469 

470 primaryKey: ddl.FieldSpec 

471 """The primary key field for this dimension (`FieldSpec`). 

472 

473 Note that the database primary keys for dimension tables are in general 

474 compound; this field is the only field in the database primary key that is 

475 not also a foreign key (to a required dependency dimension table). 

476 """ 

477 

478 alternateKeys: NamedValueSet[ddl.FieldSpec] 

479 """Additional unique key fields for this dimension that are not the the 

480 primary key (`NamedValueSet` of `FieldSpec`). 

481 """ 

482 

483 

484@immutable 

485class SkyPixDimension(Dimension): 

486 """A special `Dimension` subclass for hierarchical pixelizations of the 

487 sky. 

488 

489 Unlike most other dimensions, skypix dimension records are not stored in 

490 the database, as these records only contain an integer pixel ID and a 

491 region on the sky, and each of these can be computed directly from the 

492 other. 

493 

494 Parameters 

495 ---------- 

496 name : `str` 

497 Name of the dimension. By convention, this is a lowercase string 

498 abbreviation for the pixelization followed by its integer level, 

499 such as "htm7". 

500 pixelization : `sphgeom.Pixelization` 

501 Pixelization instance that can compute regions from IDs and IDs from 

502 points. 

503 """ 

504 

505 def __init__(self, name: str, pixelization: Pixelization): 

506 related = RelatedDimensions(required=set(), implied=set(), spatial=name) 

507 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)] 

508 super().__init__(name, related=related, uniqueKeys=uniqueKeys) 

509 self.pixelization = pixelization 

510 

511 def hasTable(self) -> bool: 

512 # Docstring inherited from DimensionElement.hasTable. 

513 return False 

514 

515 # Class attributes below are shadowed by instance attributes, and are 

516 # present just to hold the docstrings for those instance attributes. 

517 

518 pixelization: Pixelization 

519 """Pixelization instance that can compute regions from IDs and IDs from 

520 points (`sphgeom.Pixelization`). 

521 """