Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Dimension", 

26 "DimensionCombination", 

27 "DimensionElement", 

28) 

29 

30from abc import abstractmethod 

31 

32from typing import ( 

33 Any, 

34 Optional, 

35 Type, 

36 TYPE_CHECKING, 

37) 

38 

39from ..named import NamedValueAbstractSet, NamedValueSet 

40from ..utils import cached_getter 

41from .. import ddl 

42from .._topology import TopologicalRelationshipEndpoint 

43from ..json import from_json_generic, to_json_generic 

44 

45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true

46 from ._universe import DimensionUniverse 

47 from ._governor import GovernorDimension 

48 from ._graph import DimensionGraph 

49 from ._records import DimensionRecord 

50 from ...registry import Registry 

51 

52 

53class DimensionElement(TopologicalRelationshipEndpoint): 

54 """A named data-organization concept that defines a label and/or metadata 

55 in the dimensions system. 

56 

57 A `DimensionElement` instance typically corresponds to a _logical_ table in 

58 the `Registry`: either an actual database table or a way of generating rows 

59 on-the-fly that can similarly participate in queries. The rows in that 

60 table are represented by instances of a `DimensionRecord` subclass. Most 

61 `DimensionElement` instances are instances of its `Dimension` subclass, 

62 which is used for elements that can be used as data ID keys. 

63 

64 Notes 

65 ----- 

66 `DimensionElement` instances should always be constructed by and retreived 

67 from a `DimensionUniverse`. They are immutable after they are fully 

68 constructed, and should never be copied. 

69 

70 Pickling a `DimensionElement` just records its name and universe; 

71 unpickling one actually just looks up the element via the singleton 

72 dictionary of all universes. This allows pickle to be used to transfer 

73 elements between processes, but only when each process initializes its own 

74 instance of the same `DimensionUniverse`. 

75 """ 

76 

77 def __str__(self) -> str: 

78 return self.name 

79 

80 def __repr__(self) -> str: 

81 return f"{type(self).__name__}({self.name})" 

82 

83 def __eq__(self, other: Any) -> bool: 

84 try: 

85 return self.name == other.name 

86 except AttributeError: 

87 # TODO: try removing this fallback; it's not really consistent with 

88 # base class intent, and it could be confusing 

89 return self.name == other 

90 

91 def __hash__(self) -> int: 

92 return hash(self.name) 

93 

94 # TODO: try removing comparison operators; DimensionUniverse.sorted should 

95 # be adequate. 

96 

97 def __lt__(self, other: DimensionElement) -> bool: 

98 try: 

99 return self.universe.getElementIndex(self.name) < self.universe.getElementIndex(other.name) 

100 except KeyError: 

101 return NotImplemented 

102 

103 def __le__(self, other: DimensionElement) -> bool: 

104 try: 

105 return self.universe.getElementIndex(self.name) <= self.universe.getElementIndex(other.name) 

106 except KeyError: 

107 return NotImplemented 

108 

109 def __gt__(self, other: DimensionElement) -> bool: 

110 try: 

111 return self.universe.getElementIndex(self.name) > self.universe.getElementIndex(other.name) 

112 except KeyError: 

113 return NotImplemented 

114 

115 def __ge__(self, other: DimensionElement) -> bool: 

116 try: 

117 return self.universe.getElementIndex(self.name) >= self.universe.getElementIndex(other.name) 

118 except KeyError: 

119 return NotImplemented 

120 

121 @classmethod 

122 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

123 """Callable used for unpickling. 

124 

125 For internal use only. 

126 """ 

127 return universe[name] 

128 

129 def __reduce__(self) -> tuple: 

130 return (self._unpickle, (self.universe, self.name)) 

131 

132 def __deepcopy__(self, memo: dict) -> DimensionElement: 

133 # DimensionElement is recursively immutable; see note in @immutable 

134 # decorator. 

135 return self 

136 

137 def to_simple(self, minimal: bool = False) -> str: 

138 """Convert this class to a simple python type suitable for 

139 serialization. 

140 

141 Parameters 

142 ---------- 

143 minimal : `bool`, optional 

144 Use minimal serialization. Has no effect on for this class. 

145 

146 Returns 

147 ------- 

148 simple : `str` 

149 The object converted to a single string. 

150 """ 

151 return self.name 

152 

153 @classmethod 

154 def from_simple(cls, simple: str, 

155 universe: Optional[DimensionUniverse] = None, 

156 registry: Optional[Registry] = None) -> DimensionElement: 

157 """Construct a new object from the data returned from the `to_simple` 

158 method. 

159 

160 Parameters 

161 ---------- 

162 simple : `str` 

163 The value returned by `to_simple()`. 

164 universe : `DimensionUniverse` 

165 The special graph of all known dimensions. 

166 registry : `lsst.daf.butler.Registry`, optional 

167 Registry from which a universe can be extracted. Can be `None` 

168 if universe is provided explicitly. 

169 

170 Returns 

171 ------- 

172 dataId : `DimensionElement` 

173 Newly-constructed object. 

174 """ 

175 if universe is None and registry is None: 

176 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

177 if universe is None and registry is not None: 

178 universe = registry.dimensions 

179 if universe is None: 

180 # this is for mypy 

181 raise ValueError("Unable to determine a usable universe") 

182 

183 return universe[simple] 

184 

185 to_json = to_json_generic 

186 from_json = classmethod(from_json_generic) 

187 

188 def hasTable(self) -> bool: 

189 """Return `True` if this element is associated with a table 

190 (even if that table "belongs" to another element). 

191 """ 

192 return True 

193 

194 universe: DimensionUniverse 

195 """The universe of all compatible dimensions with which this element is 

196 associated (`DimensionUniverse`). 

197 """ 

198 

199 @property # type: ignore 

200 @cached_getter 

201 def governor(self) -> Optional[GovernorDimension]: 

202 """The `GovernorDimension` that is a required dependency of this 

203 element, or `None` if there is no such dimension (`GovernorDimension` 

204 or `None`). 

205 """ 

206 if len(self.graph.governors) == 1: 

207 (result,) = self.graph.governors 

208 return result 

209 elif len(self.graph.governors) > 1: 

210 raise RuntimeError( 

211 f"Dimension element {self.name} has multiple governors: {self.graph.governors}." 

212 ) 

213 else: 

214 return None 

215 

216 @property 

217 @abstractmethod 

218 def required(self) -> NamedValueAbstractSet[Dimension]: 

219 """Dimensions that are necessary to uniquely identify a record of this 

220 dimension element. 

221 

222 For elements with a database representation, these dimension are 

223 exactly those used to form the (possibly compound) primary key, and all 

224 dimensions here that are not ``self`` are also used to form foreign 

225 keys. 

226 

227 For `Dimension` instances, this should be exactly the same as 

228 ``graph.required``, but that may not be true for `DimensionElement` 

229 instances in general. When they differ, there are multiple 

230 combinations of dimensions that uniquely identify this element, but 

231 this one is more direct. 

232 """ 

233 raise NotImplementedError() 

234 

235 @property 

236 @abstractmethod 

237 def implied(self) -> NamedValueAbstractSet[Dimension]: 

238 """Other dimensions that are uniquely identified directly by a record 

239 of this dimension element. 

240 

241 For elements with a database representation, these are exactly the 

242 dimensions used to form foreign key constraints whose fields are not 

243 (wholly) also part of the primary key. 

244 

245 Unlike ``self.graph.implied``, this set is not expanded recursively. 

246 """ 

247 raise NotImplementedError() 

248 

249 @property # type: ignore 

250 @cached_getter 

251 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

252 """The union of `required` and `implied`, with all elements in 

253 `required` before any elements in `implied`. 

254 

255 This differs from ``self.graph.dimensions`` both in order and in 

256 content: 

257 

258 - as in ``self.implied``, implied dimensions are not expanded 

259 recursively here; 

260 - implied dimensions appear after required dimensions here, instead of 

261 being topologically ordered. 

262 

263 As a result, this set is ordered consistently with 

264 ``self.RecordClass.fields``. 

265 """ 

266 return NamedValueSet(list(self.required) + list(self.implied)).freeze() 

267 

268 @property # type: ignore 

269 @cached_getter 

270 def graph(self) -> DimensionGraph: 

271 """Minimal graph that includes this element (`DimensionGraph`). 

272 

273 ``self.graph.required`` includes all dimensions whose primary key 

274 values are sufficient (often necessary) to uniquely identify ``self`` 

275 (including ``self`` if ``isinstance(self, Dimension)``. 

276 ``self.graph.implied`` includes all dimensions also identified 

277 (possibly recursively) by this set. 

278 """ 

279 return self.universe.extract(self.dimensions.names) 

280 

281 @property # type: ignore 

282 @cached_getter 

283 def RecordClass(self) -> Type[DimensionRecord]: 

284 """The `DimensionRecord` subclass used to hold records for this element 

285 (`type`). 

286 

287 Because `DimensionRecord` subclasses are generated dynamically, this 

288 type cannot be imported directly and hence can only be obtained from 

289 this attribute. 

290 """ 

291 from ._records import _subclassDimensionRecord 

292 return _subclassDimensionRecord(self) 

293 

294 @property 

295 @abstractmethod 

296 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

297 """Additional metadata fields included in this element's table 

298 (`NamedValueSet` of `FieldSpec`). 

299 """ 

300 raise NotImplementedError() 

301 

302 @property 

303 def viewOf(self) -> Optional[str]: 

304 """Name of another table this element's records are drawn from (`str` 

305 or `None`). 

306 """ 

307 return None 

308 

309 @property 

310 def alwaysJoin(self) -> bool: 

311 """If `True`, always include this element in any query or data ID in 

312 which its ``required`` dimensions appear, because it defines a 

313 relationship between those dimensions that must always be satisfied. 

314 """ 

315 return False 

316 

317 

318class Dimension(DimensionElement): 

319 """A named data-organization concept that can be used as a key in a data 

320 ID. 

321 """ 

322 

323 @property 

324 @abstractmethod 

325 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

326 """All fields that can individually be used to identify records of this 

327 element, given the primary keys of all required dependencies 

328 (`NamedValueAbstractSet` of `FieldSpec`). 

329 """ 

330 raise NotImplementedError() 

331 

332 @property # type: ignore 

333 @cached_getter 

334 def primaryKey(self) -> ddl.FieldSpec: 

335 """The primary key field for this dimension (`FieldSpec`). 

336 

337 Note that the database primary keys for dimension tables are in general 

338 compound; this field is the only field in the database primary key that 

339 is not also a foreign key (to a required dependency dimension table). 

340 """ 

341 primaryKey, *_ = self.uniqueKeys 

342 return primaryKey 

343 

344 @property # type: ignore 

345 @cached_getter 

346 def alternateKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

347 """Additional unique key fields for this dimension that are not the 

348 primary key (`NamedValueAbstractSet` of `FieldSpec`). 

349 

350 If this dimension has required dependencies, the keys of those 

351 dimensions are also included in the unique constraints defined for 

352 these alternate keys. 

353 """ 

354 _, *alternateKeys = self.uniqueKeys 

355 return NamedValueSet(alternateKeys).freeze() 

356 

357 

358class DimensionCombination(DimensionElement): 

359 """A `DimensionElement` that provides extra metadata and/or relationship 

360 endpoint information for a combination of dimensions. 

361 """ 

362 pass