Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Dimension", 

26 "DimensionCombination", 

27 "DimensionElement", 

28) 

29 

30from abc import abstractmethod 

31 

32from typing import ( 

33 Any, 

34 Optional, 

35 Type, 

36 TYPE_CHECKING, 

37) 

38 

39from ..named import NamedValueAbstractSet, NamedValueSet 

40from ..utils import cached_getter 

41from .. import ddl 

42from .._topology import TopologicalRelationshipEndpoint 

43from ..json import from_json_generic, to_json_generic 

44 

45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true

46 from ._universe import DimensionUniverse 

47 from ._governor import GovernorDimension 

48 from ._graph import DimensionGraph 

49 from ._records import DimensionRecord 

50 from ...registry import Registry 

51 

52 

53class DimensionElement(TopologicalRelationshipEndpoint): 

54 """A label and/or metadata in the dimensions system. 

55 

56 A named data-organization concept that defines a label and/or metadata 

57 in the dimensions system. 

58 

59 A `DimensionElement` instance typically corresponds to a _logical_ table in 

60 the `Registry`: either an actual database table or a way of generating rows 

61 on-the-fly that can similarly participate in queries. The rows in that 

62 table are represented by instances of a `DimensionRecord` subclass. Most 

63 `DimensionElement` instances are instances of its `Dimension` subclass, 

64 which is used for elements that can be used as data ID keys. 

65 

66 Notes 

67 ----- 

68 `DimensionElement` instances should always be constructed by and retreived 

69 from a `DimensionUniverse`. They are immutable after they are fully 

70 constructed, and should never be copied. 

71 

72 Pickling a `DimensionElement` just records its name and universe; 

73 unpickling one actually just looks up the element via the singleton 

74 dictionary of all universes. This allows pickle to be used to transfer 

75 elements between processes, but only when each process initializes its own 

76 instance of the same `DimensionUniverse`. 

77 """ 

78 

79 def __str__(self) -> str: 

80 return self.name 

81 

82 def __repr__(self) -> str: 

83 return f"{type(self).__name__}({self.name})" 

84 

85 def __eq__(self, other: Any) -> bool: 

86 try: 

87 return self.name == other.name 

88 except AttributeError: 

89 # TODO: try removing this fallback; it's not really consistent with 

90 # base class intent, and it could be confusing 

91 return self.name == other 

92 

93 def __hash__(self) -> int: 

94 return hash(self.name) 

95 

96 # TODO: try removing comparison operators; DimensionUniverse.sorted should 

97 # be adequate. 

98 

99 def __lt__(self, other: DimensionElement) -> bool: 

100 try: 

101 return self.universe.getElementIndex(self.name) < self.universe.getElementIndex(other.name) 

102 except KeyError: 

103 return NotImplemented 

104 

105 def __le__(self, other: DimensionElement) -> bool: 

106 try: 

107 return self.universe.getElementIndex(self.name) <= self.universe.getElementIndex(other.name) 

108 except KeyError: 

109 return NotImplemented 

110 

111 def __gt__(self, other: DimensionElement) -> bool: 

112 try: 

113 return self.universe.getElementIndex(self.name) > self.universe.getElementIndex(other.name) 

114 except KeyError: 

115 return NotImplemented 

116 

117 def __ge__(self, other: DimensionElement) -> bool: 

118 try: 

119 return self.universe.getElementIndex(self.name) >= self.universe.getElementIndex(other.name) 

120 except KeyError: 

121 return NotImplemented 

122 

123 @classmethod 

124 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

125 """Callable used for unpickling. 

126 

127 For internal use only. 

128 """ 

129 return universe[name] 

130 

131 def __reduce__(self) -> tuple: 

132 return (self._unpickle, (self.universe, self.name)) 

133 

134 def __deepcopy__(self, memo: dict) -> DimensionElement: 

135 # DimensionElement is recursively immutable; see note in @immutable 

136 # decorator. 

137 return self 

138 

139 def to_simple(self, minimal: bool = False) -> str: 

140 """Convert this class to a simple python type. 

141 

142 This is suitable for serialization. 

143 

144 Parameters 

145 ---------- 

146 minimal : `bool`, optional 

147 Use minimal serialization. Has no effect on for this class. 

148 

149 Returns 

150 ------- 

151 simple : `str` 

152 The object converted to a single string. 

153 """ 

154 return self.name 

155 

156 @classmethod 

157 def from_simple(cls, simple: str, 

158 universe: Optional[DimensionUniverse] = None, 

159 registry: Optional[Registry] = None) -> DimensionElement: 

160 """Construct a new object from the simplified form. 

161 

162 Usually the data is returned from the `to_simple` method. 

163 

164 Parameters 

165 ---------- 

166 simple : `str` 

167 The value returned by `to_simple()`. 

168 universe : `DimensionUniverse` 

169 The special graph of all known dimensions. 

170 registry : `lsst.daf.butler.Registry`, optional 

171 Registry from which a universe can be extracted. Can be `None` 

172 if universe is provided explicitly. 

173 

174 Returns 

175 ------- 

176 dataId : `DimensionElement` 

177 Newly-constructed object. 

178 """ 

179 if universe is None and registry is None: 

180 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

181 if universe is None and registry is not None: 

182 universe = registry.dimensions 

183 if universe is None: 

184 # this is for mypy 

185 raise ValueError("Unable to determine a usable universe") 

186 

187 return universe[simple] 

188 

189 to_json = to_json_generic 

190 from_json = classmethod(from_json_generic) 

191 

192 def hasTable(self) -> bool: 

193 """Indicate if this element is associated with a table. 

194 

195 Return `True` if this element is associated with a table 

196 (even if that table "belongs" to another element). 

197 """ 

198 return True 

199 

200 universe: DimensionUniverse 

201 """The universe of all compatible dimensions with which this element is 

202 associated (`DimensionUniverse`). 

203 """ 

204 

205 @property # type: ignore 

206 @cached_getter 

207 def governor(self) -> Optional[GovernorDimension]: 

208 """Return the governor dimension. 

209 

210 This is the `GovernorDimension` that is a required dependency of this 

211 element, or `None` if there is no such dimension (`GovernorDimension` 

212 or `None`). 

213 """ 

214 if len(self.graph.governors) == 1: 

215 (result,) = self.graph.governors 

216 return result 

217 elif len(self.graph.governors) > 1: 

218 raise RuntimeError( 

219 f"Dimension element {self.name} has multiple governors: {self.graph.governors}." 

220 ) 

221 else: 

222 return None 

223 

224 @property 

225 @abstractmethod 

226 def required(self) -> NamedValueAbstractSet[Dimension]: 

227 """Return the required dimensions. 

228 

229 Dimensions that are necessary to uniquely identify a record of this 

230 dimension element. 

231 

232 For elements with a database representation, these dimension are 

233 exactly those used to form the (possibly compound) primary key, and all 

234 dimensions here that are not ``self`` are also used to form foreign 

235 keys. 

236 

237 For `Dimension` instances, this should be exactly the same as 

238 ``graph.required``, but that may not be true for `DimensionElement` 

239 instances in general. When they differ, there are multiple 

240 combinations of dimensions that uniquely identify this element, but 

241 this one is more direct. 

242 """ 

243 raise NotImplementedError() 

244 

245 @property 

246 @abstractmethod 

247 def implied(self) -> NamedValueAbstractSet[Dimension]: 

248 """Return the implied dimensions. 

249 

250 Other dimensions that are uniquely identified directly by a record 

251 of this dimension element. 

252 

253 For elements with a database representation, these are exactly the 

254 dimensions used to form foreign key constraints whose fields are not 

255 (wholly) also part of the primary key. 

256 

257 Unlike ``self.graph.implied``, this set is not expanded recursively. 

258 """ 

259 raise NotImplementedError() 

260 

261 @property # type: ignore 

262 @cached_getter 

263 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

264 """Return all dimensions. 

265 

266 The union of `required` and `implied`, with all elements in 

267 `required` before any elements in `implied`. 

268 

269 This differs from ``self.graph.dimensions`` both in order and in 

270 content: 

271 

272 - as in ``self.implied``, implied dimensions are not expanded 

273 recursively here; 

274 - implied dimensions appear after required dimensions here, instead of 

275 being topologically ordered. 

276 

277 As a result, this set is ordered consistently with 

278 ``self.RecordClass.fields``. 

279 """ 

280 return NamedValueSet(list(self.required) + list(self.implied)).freeze() 

281 

282 @property # type: ignore 

283 @cached_getter 

284 def graph(self) -> DimensionGraph: 

285 """Return minimal graph that includes this element (`DimensionGraph`). 

286 

287 ``self.graph.required`` includes all dimensions whose primary key 

288 values are sufficient (often necessary) to uniquely identify ``self`` 

289 (including ``self`` if ``isinstance(self, Dimension)``. 

290 ``self.graph.implied`` includes all dimensions also identified 

291 (possibly recursively) by this set. 

292 """ 

293 return self.universe.extract(self.dimensions.names) 

294 

295 @property # type: ignore 

296 @cached_getter 

297 def RecordClass(self) -> Type[DimensionRecord]: 

298 """Return the record subclass for this element. 

299 

300 The `DimensionRecord` subclass used to hold records for this element 

301 (`type`). 

302 

303 Because `DimensionRecord` subclasses are generated dynamically, this 

304 type cannot be imported directly and hence can only be obtained from 

305 this attribute. 

306 """ 

307 from ._records import _subclassDimensionRecord 

308 return _subclassDimensionRecord(self) 

309 

310 @property 

311 @abstractmethod 

312 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

313 """Additional metadata fields included in this element's table. 

314 

315 (`NamedValueSet` of `FieldSpec`). 

316 """ 

317 raise NotImplementedError() 

318 

319 @property 

320 def viewOf(self) -> Optional[str]: 

321 """Name of another table this element's records are drawn from. 

322 

323 (`str` or `None`). 

324 """ 

325 return None 

326 

327 @property 

328 def alwaysJoin(self) -> bool: 

329 """Indicate if the element should always be included. 

330 

331 If `True`, always include this element in any query or data ID in 

332 which its ``required`` dimensions appear, because it defines a 

333 relationship between those dimensions that must always be satisfied. 

334 """ 

335 return False 

336 

337 

338class Dimension(DimensionElement): 

339 """A dimension. 

340 

341 A named data-organization concept that can be used as a key in a data 

342 ID. 

343 """ 

344 

345 @property 

346 @abstractmethod 

347 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

348 """Return the unique fields. 

349 

350 All fields that can individually be used to identify records of this 

351 element, given the primary keys of all required dependencies 

352 (`NamedValueAbstractSet` of `FieldSpec`). 

353 """ 

354 raise NotImplementedError() 

355 

356 @property # type: ignore 

357 @cached_getter 

358 def primaryKey(self) -> ddl.FieldSpec: 

359 """Return primary key field for this dimension (`FieldSpec`). 

360 

361 Note that the database primary keys for dimension tables are in general 

362 compound; this field is the only field in the database primary key that 

363 is not also a foreign key (to a required dependency dimension table). 

364 """ 

365 primaryKey, *_ = self.uniqueKeys 

366 return primaryKey 

367 

368 @property # type: ignore 

369 @cached_getter 

370 def alternateKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

371 """Return alternate keys. 

372 

373 Additional unique key fields for this dimension that are not the 

374 primary key (`NamedValueAbstractSet` of `FieldSpec`). 

375 

376 If this dimension has required dependencies, the keys of those 

377 dimensions are also included in the unique constraints defined for 

378 these alternate keys. 

379 """ 

380 _, *alternateKeys = self.uniqueKeys 

381 return NamedValueSet(alternateKeys).freeze() 

382 

383 

384class DimensionCombination(DimensionElement): 

385 """Element with extra information. 

386 

387 A `DimensionElement` that provides extra metadata and/or relationship 

388 endpoint information for a combination of dimensions. 

389 """