Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionElement", "Dimension", "SkyPixDimension"] 

25 

26from typing import Optional, Iterable, AbstractSet, TYPE_CHECKING 

27 

28from sqlalchemy import Integer 

29 

30from lsst.sphgeom import Pixelization 

31from ..utils import NamedValueSet, immutable 

32from .. import ddl 

33from .records import _subclassDimensionRecord 

34from .graph import DimensionGraph 

35 

36if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 36 ↛ 37line 36 didn't jump to line 37, because the condition on line 36 was never true

37 from .universe import DimensionUniverse 

38 

39 

40@immutable 

41class DimensionElement: 

42 """A named data-organization concept that defines a label and/or metadata 

43 in the dimensions system. 

44 

45 A `DimensionElement` instance typically corresponds to a table in the 

46 `Registry`; the rows in that table are represented by instances of a 

47 `DimensionRecord` subclass. Most `DimensionElement` instances are 

48 instances of its `Dimension` subclass, which is used for elements that can 

49 be used as data ID keys. The base class itself can be used for other 

50 dimension tables that provide metadata keyed by true dimensions. 

51 

52 Parameters 

53 ---------- 

54 name : `str` 

55 Name of the element. Used as at least part of the table name, if 

56 the dimension in associated with a database table. 

57 directDependencyNames : iterable of `str` 

58 The names of all dimensions this elements depends on directly, 

59 including both required dimensions (those needed to identify a 

60 record of this element) an implied dimensions. 

61 impliedDependencyNames : iterable of `str` 

62 The names of all dimensions that are identified by records of this 

63 element, but are not needed to identify it. 

64 spatial : `bool` 

65 Whether records of this element are associated with a region on the 

66 sky. 

67 temporal : `bool` 

68 Whether records of this element are associated with a timespan. 

69 metadata : iterable of `FieldSpec` 

70 Additional metadata fields included in this element's table. 

71 cached : `bool` 

72 Whether `Registry` should cache records of this element in-memory. 

73 viewOf : `str`, optional 

74 Name of another table this element's records should be drawn from. The 

75 fields of this table must be a superset of the fields of the element. 

76 

77 Notes 

78 ----- 

79 `DimensionElement` instances should always be constructed by and retreived 

80 from a `DimensionUniverse`. They are immutable after they are fully 

81 constructed, and should never be copied. 

82 

83 Pickling a `DimensionElement` just records its name and universe; 

84 unpickling one actually just looks up the element via the singleton 

85 dictionary of all universes. This allows pickle to be used to transfer 

86 elements between processes, but only when each process initializes its own 

87 instance of the same `DimensionUniverse`. 

88 """ 

89 

90 def __init__(self, name: str, *, 

91 directDependencyNames: Iterable[str] = (), 

92 impliedDependencyNames: Iterable[str] = (), 

93 spatial: bool = False, 

94 temporal: bool = False, 

95 metadata: Iterable[ddl.FieldSpec] = (), 

96 cached: bool = False, 

97 viewOf: Optional[str] = None): 

98 self.name = name 

99 self._directDependencyNames = frozenset(directDependencyNames) 

100 self._impliedDependencyNames = frozenset(impliedDependencyNames) 

101 self.spatial = spatial 

102 self.temporal = temporal 

103 self.metadata = NamedValueSet(metadata) 

104 self.metadata.freeze() 

105 self.cached = cached 

106 self.viewOf = viewOf 

107 

108 def _finish(self, universe: DimensionUniverse): 

109 """Finish construction of the element and add it to the given universe. 

110 

111 For internal use by `DimensionUniverse` only. 

112 """ 

113 # Attach set self.universe and add self to universe attributes; 

114 # let subclasses override which attributes by calling a separate 

115 # method. 

116 self._attachToUniverse(universe) 

117 # Expand direct dependencies into recursive dependencies. 

118 expanded = set(self._directDependencyNames) 

119 for name in self._directDependencyNames: 

120 expanded.update(universe[name]._recursiveDependencyNames) 

121 self._recursiveDependencyNames = frozenset(expanded) 

122 # Define self.implied, a public, sorted version of 

123 # self._impliedDependencyNames. 

124 self.implied = NamedValueSet(universe.sorted(self._impliedDependencyNames)) 

125 self.implied.freeze() 

126 # Attach a DimensionGraph that provides the public API for getting 

127 # at requirements. Again delegate to subclasses. 

128 self._attachGraph() 

129 # Create and attach a DimensionRecord subclass to hold values of this 

130 # dimension type. 

131 self.RecordClass = _subclassDimensionRecord(self) 

132 

133 def _attachToUniverse(self, universe: DimensionUniverse): 

134 """Add the element to the given universe. 

135 

136 Called only by `_finish`, but may be overridden by subclasses. 

137 """ 

138 self.universe = universe 

139 self.universe.elements.add(self) 

140 

141 def _attachGraph(self): 

142 """Initialize the `graph` attribute for this element. 

143 

144 Called only by `_finish`, but may be overridden by subclasses. 

145 """ 

146 from .graph import DimensionGraph 

147 self.graph = DimensionGraph(self.universe, names=self._recursiveDependencyNames, conform=False) 

148 

149 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]): 

150 """Return `True` if this element should be included in `DimensionGraph` 

151 that includes the named dimensions. 

152 

153 For internal use by `DimensionGraph` only. 

154 """ 

155 return self._directDependencyNames.issubset(dimensionNames) 

156 

157 def __str__(self) -> str: 

158 return self.name 

159 

160 def __repr__(self) -> str: 

161 return f"{type(self).__name__}({self.name})" 

162 

163 def __eq__(self, other) -> bool: 

164 try: 

165 return self.name == other.name 

166 except AttributeError: 

167 return self.name == other 

168 

169 def __hash__(self) -> int: 

170 return hash(self.name) 

171 

172 def __lt__(self, other) -> bool: 

173 try: 

174 return self.universe._elementIndices[self] < self.universe._elementIndices[other] 

175 except KeyError: 

176 return NotImplemented 

177 

178 def __le__(self, other) -> bool: 

179 try: 

180 return self.universe._elementIndices[self] <= self.universe._elementIndices[other] 

181 except KeyError: 

182 return NotImplemented 

183 

184 def __gt__(self, other) -> bool: 

185 try: 

186 return self.universe._elementIndices[self] > self.universe._elementIndices[other] 

187 except KeyError: 

188 return NotImplemented 

189 

190 def __ge__(self, other) -> bool: 

191 try: 

192 return self.universe._elementIndices[self] >= self.universe._elementIndices[other] 

193 except KeyError: 

194 return NotImplemented 

195 

196 def hasTable(self) -> bool: 

197 """Return `True` if this element is associated with a table 

198 (even if that table "belongs" to another element). 

199 

200 Instances of the `DimensionElement` base class itself are always 

201 associated with tables. 

202 """ 

203 return True 

204 

205 @classmethod 

206 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

207 """Callable used for unpickling. 

208 

209 For internal use only. 

210 """ 

211 return universe.elements[name] 

212 

213 def __reduce__(self) -> tuple: 

214 return (self._unpickle, (self.universe, self.name)) 

215 

216 # Class attributes below are shadowed by instance attributes, and are 

217 # present just to hold the docstrings for those instance attributes. 

218 

219 universe: DimensionUniverse 

220 """The universe of all compatible dimensions with which this element is 

221 associated (`DimensionUniverse`). 

222 """ 

223 

224 name: str 

225 """Unique name for this dimension element (`str`). 

226 """ 

227 

228 graph: DimensionGraph 

229 """Minimal graph that includes this element (`DimensionGraph`). 

230 

231 ``self.graph.required`` includes all dimensions whose primary key values 

232 must be provided in order to uniquely identify ``self`` (including ``self`` 

233 if ``isinstance(self, Dimension)`. ``self.graph.implied`` includes all 

234 dimensions also identified (possibly recursively) by this set. 

235 """ 

236 

237 implied: NamedValueSet[Dimension] 

238 """Other dimensions that are uniquely identified directly by a record of 

239 this dimension. 

240 

241 Unlike ``self.graph.implied``, this set is not expanded recursively. 

242 """ 

243 

244 spatial: bool 

245 """Whether records of this element are associated with a region on the sky 

246 (`bool`). 

247 """ 

248 

249 temporal: bool 

250 """Whether records of this element are associated with a timespan (`bool`). 

251 """ 

252 

253 metadata: NamedValueSet[ddl.FieldSpec] 

254 """Additional metadata fields included in this element's table 

255 (`NamedValueSet` of `FieldSpec`). 

256 """ 

257 

258 RecordClass: type 

259 """The `DimensionRecord` subclass used to hold records for this element 

260 (`type`). 

261 

262 Because `DimensionRecord` subclasses are generated dynamically, this type 

263 cannot be imported directly and hence canonly be obtained from this 

264 attribute. 

265 """ 

266 

267 cached: bool 

268 """Whether `Registry` should cache records of this element in-memory 

269 (`bool`). 

270 """ 

271 

272 viewOf: Optional[str] 

273 """Name of another table this elements records are drawn from (`str` or 

274 `None`). 

275 """ 

276 

277 

278@immutable 

279class Dimension(DimensionElement): 

280 """A named data-organization concept that can be used as a key in a data 

281 ID. 

282 

283 Parameters 

284 ---------- 

285 name : `str` 

286 Name of the dimension. Used as at least part of the table name, if 

287 the dimension in associated with a database table, and as an alternate 

288 key (instead of the instance itself) in data IDs. 

289 uniqueKeys : iterable of `FieldSpec` 

290 Fields that can *each* be used to uniquely identify this dimension 

291 (once all required dependencies are also identified). The first entry 

292 will be used as the table's primary key and as a foriegn key field in 

293 the fields of dependent tables. 

294 kwds 

295 Additional keyword arguments are forwarded to the `DimensionElement` 

296 constructor. 

297 """ 

298 

299 def __init__(self, name: str, *, uniqueKeys: Iterable[ddl.FieldSpec], **kwds): 

300 super().__init__(name, **kwds) 

301 self.uniqueKeys = NamedValueSet(uniqueKeys) 

302 self.uniqueKeys.freeze() 

303 self.primaryKey, *alternateKeys = uniqueKeys 

304 self.alternateKeys = NamedValueSet(alternateKeys) 

305 self.alternateKeys.freeze() 

306 

307 def _attachToUniverse(self, universe: DimensionUniverse): 

308 # Docstring inherited from DimensionElement._attachToUniverse. 

309 super()._attachToUniverse(universe) 

310 universe.dimensions.add(self) 

311 

312 def _attachGraph(self): 

313 # Docstring inherited from DimensionElement._attachGraph. 

314 self.graph = DimensionGraph(self.universe, 

315 names=self._recursiveDependencyNames.union([self.name]), 

316 conform=False) 

317 

318 def _shouldBeInGraph(self, dimensionNames: AbstractSet[str]): 

319 # Docstring inherited from DimensionElement._shouldBeInGraph. 

320 return self.name in dimensionNames 

321 

322 # Class attributes below are shadowed by instance attributes, and are 

323 # present just to hold the docstrings for those instance attributes. 

324 

325 uniqueKeys: NamedValueSet[ddl.FieldSpec] 

326 """All fields that can individually be used to identify records of this 

327 element, given the primary keys of all required dependencies 

328 (`NamedValueSet` of `FieldSpec`). 

329 """ 

330 

331 primaryKey: ddl.FieldSpec 

332 """The primary key field for this dimension (`FieldSpec`). 

333 

334 Note that the database primary keys for dimension tables are in general 

335 compound; this field is the only field in the database primary key that is 

336 not also a foreign key (to a required dependency dimension table). 

337 """ 

338 

339 alternateKeys: NamedValueSet[ddl.FieldSpec] 

340 """Additional unique key fields for this dimension that are not the the 

341 primary key (`NamedValueSet` of `FieldSpec`). 

342 """ 

343 

344 

345@immutable 

346class SkyPixDimension(Dimension): 

347 """A special `Dimension` subclass for hierarchical pixelizations of the 

348 sky. 

349 

350 Unlike most other dimensions, skypix dimension records are not stored in 

351 the database, as these records only contain an integer pixel ID and a 

352 region on the sky, and each of these can be computed directly from the 

353 other. 

354 

355 Parameters 

356 ---------- 

357 name : `str` 

358 Name of the dimension. By convention, this is a lowercase string 

359 abbreviation for the pixelization followed by its integer level, 

360 such as "htm7". 

361 pixelization : `sphgeom.Pixelization` 

362 Pixelization instance that can compute regions from IDs and IDs from 

363 points. 

364 """ 

365 

366 def __init__(self, name: str, pixelization: Pixelization): 

367 uniqueKeys = [ddl.FieldSpec(name="id", dtype=Integer, primaryKey=True, nullable=False)] 

368 super().__init__(name, uniqueKeys=uniqueKeys, spatial=True) 

369 self.pixelization = pixelization 

370 

371 def hasTable(self) -> bool: 

372 # Docstring inherited from DimensionElement.hasTable. 

373 return False 

374 

375 # Class attributes below are shadowed by instance attributes, and are 

376 # present just to hold the docstrings for those instance attributes. 

377 

378 pixelization: Pixelization 

379 """Pixelization instance that can compute regions from IDs and IDs from 

380 points (`sphgeom.Pixelization`). 

381 """