Coverage for python/lsst/daf/butler/core/dimensions/_elements.py: 52%

117 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-23 09:30 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Dimension", 

26 "DimensionCombination", 

27 "DimensionElement", 

28) 

29 

30from abc import abstractmethod 

31from typing import TYPE_CHECKING, Any, ClassVar 

32 

33from lsst.utils.classes import cached_getter 

34 

35from .. import ddl 

36from .._topology import TopologicalRelationshipEndpoint 

37from ..json import from_json_generic, to_json_generic 

38from ..named import NamedValueAbstractSet, NamedValueSet 

39 

40if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

41 from ...registry import Registry 

42 from ._governor import GovernorDimension 

43 from ._graph import DimensionGraph 

44 from ._records import DimensionRecord 

45 from ._universe import DimensionUniverse 

46 

47 

48class DimensionElement(TopologicalRelationshipEndpoint): 

49 """A label and/or metadata in the dimensions system. 

50 

51 A named data-organization concept that defines a label and/or metadata 

52 in the dimensions system. 

53 

54 A `DimensionElement` instance typically corresponds to a _logical_ table in 

55 the `Registry`: either an actual database table or a way of generating rows 

56 on-the-fly that can similarly participate in queries. The rows in that 

57 table are represented by instances of a `DimensionRecord` subclass. Most 

58 `DimensionElement` instances are instances of its `Dimension` subclass, 

59 which is used for elements that can be used as data ID keys. 

60 

61 Notes 

62 ----- 

63 `DimensionElement` instances should always be constructed by and retrieved 

64 from a `DimensionUniverse`. They are immutable after they are fully 

65 constructed, and should never be copied. 

66 

67 Pickling a `DimensionElement` just records its name and universe; 

68 unpickling one actually just looks up the element via the singleton 

69 dictionary of all universes. This allows pickle to be used to transfer 

70 elements between processes, but only when each process initializes its own 

71 instance of the same `DimensionUniverse`. 

72 """ 

73 

74 def __str__(self) -> str: 

75 return self.name 

76 

77 def __repr__(self) -> str: 

78 return f"{type(self).__name__}({self.name})" 

79 

80 def __eq__(self, other: Any) -> bool: 

81 try: 

82 return self.name == other.name 

83 except AttributeError: 

84 # TODO: try removing this fallback; it's not really consistent with 

85 # base class intent, and it could be confusing 

86 return self.name == other 

87 

88 def __hash__(self) -> int: 

89 return hash(self.name) 

90 

91 # TODO: try removing comparison operators; DimensionUniverse.sorted should 

92 # be adequate. 

93 

94 def __lt__(self, other: DimensionElement) -> bool: 

95 try: 

96 return self.universe.getElementIndex(self.name) < self.universe.getElementIndex(other.name) 

97 except KeyError: 

98 return NotImplemented 

99 

100 def __le__(self, other: DimensionElement) -> bool: 

101 try: 

102 return self.universe.getElementIndex(self.name) <= self.universe.getElementIndex(other.name) 

103 except KeyError: 

104 return NotImplemented 

105 

106 def __gt__(self, other: DimensionElement) -> bool: 

107 try: 

108 return self.universe.getElementIndex(self.name) > self.universe.getElementIndex(other.name) 

109 except KeyError: 

110 return NotImplemented 

111 

112 def __ge__(self, other: DimensionElement) -> bool: 

113 try: 

114 return self.universe.getElementIndex(self.name) >= self.universe.getElementIndex(other.name) 

115 except KeyError: 

116 return NotImplemented 

117 

118 @classmethod 

119 def _unpickle(cls, universe: DimensionUniverse, name: str) -> DimensionElement: 

120 """Callable used for unpickling. 

121 

122 For internal use only. 

123 """ 

124 return universe[name] 

125 

126 def __reduce__(self) -> tuple: 

127 return (self._unpickle, (self.universe, self.name)) 

128 

129 def __deepcopy__(self, memo: dict) -> DimensionElement: 

130 # DimensionElement is recursively immutable; see note in @immutable 

131 # decorator. 

132 return self 

133 

134 def to_simple(self, minimal: bool = False) -> str: 

135 """Convert this class to a simple python type. 

136 

137 This is suitable for serialization. 

138 

139 Parameters 

140 ---------- 

141 minimal : `bool`, optional 

142 Use minimal serialization. Has no effect on for this class. 

143 

144 Returns 

145 ------- 

146 simple : `str` 

147 The object converted to a single string. 

148 """ 

149 return self.name 

150 

151 @classmethod 

152 def from_simple( 

153 cls, simple: str, universe: DimensionUniverse | None = None, registry: Registry | None = None 

154 ) -> DimensionElement: 

155 """Construct a new object from the simplified form. 

156 

157 Usually the data is returned from the `to_simple` method. 

158 

159 Parameters 

160 ---------- 

161 simple : `str` 

162 The value returned by `to_simple()`. 

163 universe : `DimensionUniverse` 

164 The special graph of all known dimensions. 

165 registry : `lsst.daf.butler.Registry`, optional 

166 Registry from which a universe can be extracted. Can be `None` 

167 if universe is provided explicitly. 

168 

169 Returns 

170 ------- 

171 dataId : `DimensionElement` 

172 Newly-constructed object. 

173 """ 

174 if universe is None and registry is None: 

175 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

176 if universe is None and registry is not None: 

177 universe = registry.dimensions 

178 if universe is None: 

179 # this is for mypy 

180 raise ValueError("Unable to determine a usable universe") 

181 

182 return universe[simple] 

183 

184 to_json = to_json_generic 

185 from_json: ClassVar = classmethod(from_json_generic) 

186 

187 def hasTable(self) -> bool: 

188 """Indicate if this element is associated with a table. 

189 

190 Return `True` if this element is associated with a table 

191 (even if that table "belongs" to another element). 

192 """ 

193 return True 

194 

195 universe: DimensionUniverse 

196 """The universe of all compatible dimensions with which this element is 

197 associated (`DimensionUniverse`). 

198 """ 

199 

200 @property 

201 @cached_getter 

202 def governor(self) -> GovernorDimension | None: 

203 """Return the governor dimension. 

204 

205 This is the `GovernorDimension` that is a required dependency of this 

206 element, or `None` if there is no such dimension (`GovernorDimension` 

207 or `None`). 

208 """ 

209 if len(self.graph.governors) == 1: 

210 (result,) = self.graph.governors 

211 return result 

212 elif len(self.graph.governors) > 1: 

213 raise RuntimeError( 

214 f"Dimension element {self.name} has multiple governors: {self.graph.governors}." 

215 ) 

216 else: 

217 return None 

218 

219 @property 

220 @abstractmethod 

221 def required(self) -> NamedValueAbstractSet[Dimension]: 

222 """Return the required dimensions. 

223 

224 Dimensions that are necessary to uniquely identify a record of this 

225 dimension element. 

226 

227 For elements with a database representation, these dimension are 

228 exactly those used to form the (possibly compound) primary key, and all 

229 dimensions here that are not ``self`` are also used to form foreign 

230 keys. 

231 

232 For `Dimension` instances, this should be exactly the same as 

233 ``graph.required``, but that may not be true for `DimensionElement` 

234 instances in general. When they differ, there are multiple 

235 combinations of dimensions that uniquely identify this element, but 

236 this one is more direct. 

237 """ 

238 raise NotImplementedError() 

239 

240 @property 

241 @abstractmethod 

242 def implied(self) -> NamedValueAbstractSet[Dimension]: 

243 """Return the implied dimensions. 

244 

245 Other dimensions that are uniquely identified directly by a record 

246 of this dimension element. 

247 

248 For elements with a database representation, these are exactly the 

249 dimensions used to form foreign key constraints whose fields are not 

250 (wholly) also part of the primary key. 

251 

252 Unlike ``self.graph.implied``, this set is not expanded recursively. 

253 """ 

254 raise NotImplementedError() 

255 

256 @property 

257 @cached_getter 

258 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

259 """Return all dimensions. 

260 

261 The union of `required` and `implied`, with all elements in 

262 `required` before any elements in `implied`. 

263 

264 This differs from ``self.graph.dimensions`` both in order and in 

265 content: 

266 

267 - as in ``self.implied``, implied dimensions are not expanded 

268 recursively here; 

269 - implied dimensions appear after required dimensions here, instead of 

270 being topologically ordered. 

271 

272 As a result, this set is ordered consistently with 

273 ``self.RecordClass.fields``. 

274 """ 

275 return NamedValueSet(list(self.required) + list(self.implied)).freeze() 

276 

277 @property 

278 @cached_getter 

279 def graph(self) -> DimensionGraph: 

280 """Return minimal graph that includes this element (`DimensionGraph`). 

281 

282 ``self.graph.required`` includes all dimensions whose primary key 

283 values are sufficient (often necessary) to uniquely identify ``self`` 

284 (including ``self`` if ``isinstance(self, Dimension)``. 

285 ``self.graph.implied`` includes all dimensions also identified 

286 (possibly recursively) by this set. 

287 """ 

288 return self.universe.extract(self.dimensions.names) 

289 

290 @property 

291 @cached_getter 

292 def RecordClass(self) -> type[DimensionRecord]: 

293 """Return the record subclass for this element. 

294 

295 The `DimensionRecord` subclass used to hold records for this element 

296 (`type`). 

297 

298 Because `DimensionRecord` subclasses are generated dynamically, this 

299 type cannot be imported directly and hence can only be obtained from 

300 this attribute. 

301 """ 

302 from ._records import _subclassDimensionRecord 

303 

304 return _subclassDimensionRecord(self) 

305 

306 @property 

307 @abstractmethod 

308 def metadata(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

309 """Additional metadata fields included in this element's table. 

310 

311 (`NamedValueSet` of `FieldSpec`). 

312 """ 

313 raise NotImplementedError() 

314 

315 @property 

316 def viewOf(self) -> str | None: 

317 """Name of another table this element's records are drawn from. 

318 

319 (`str` or `None`). 

320 """ 

321 return None 

322 

323 @property 

324 def alwaysJoin(self) -> bool: 

325 """Indicate if the element should always be included. 

326 

327 If `True`, always include this element in any query or data ID in 

328 which its ``required`` dimensions appear, because it defines a 

329 relationship between those dimensions that must always be satisfied. 

330 """ 

331 return False 

332 

333 

334class Dimension(DimensionElement): 

335 """A dimension. 

336 

337 A named data-organization concept that can be used as a key in a data 

338 ID. 

339 """ 

340 

341 @property 

342 @abstractmethod 

343 def uniqueKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

344 """Return the unique fields. 

345 

346 All fields that can individually be used to identify records of this 

347 element, given the primary keys of all required dependencies 

348 (`NamedValueAbstractSet` of `FieldSpec`). 

349 """ 

350 raise NotImplementedError() 

351 

352 @property 

353 @cached_getter 

354 def primaryKey(self) -> ddl.FieldSpec: 

355 """Return primary key field for this dimension (`FieldSpec`). 

356 

357 Note that the database primary keys for dimension tables are in general 

358 compound; this field is the only field in the database primary key that 

359 is not also a foreign key (to a required dependency dimension table). 

360 """ 

361 primaryKey, *_ = self.uniqueKeys 

362 return primaryKey 

363 

364 @property 

365 @cached_getter 

366 def alternateKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]: 

367 """Return alternate keys. 

368 

369 Additional unique key fields for this dimension that are not the 

370 primary key (`NamedValueAbstractSet` of `FieldSpec`). 

371 

372 If this dimension has required dependencies, the keys of those 

373 dimensions are also included in the unique constraints defined for 

374 these alternate keys. 

375 """ 

376 _, *alternateKeys = self.uniqueKeys 

377 return NamedValueSet(alternateKeys).freeze() 

378 

379 

380class DimensionCombination(DimensionElement): 

381 """Element with extra information. 

382 

383 A `DimensionElement` that provides extra metadata and/or relationship 

384 endpoint information for a combination of dimensions. 

385 """