Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

# This file is part of daf_butler. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

from __future__ import annotations 

 

__all__ = ["DimensionUniverse"] 

 

import pickle 

from typing import Optional, Iterable, List, Dict, Union, TYPE_CHECKING 

 

from ..config import Config 

from ..utils import NamedValueSet, immutable 

from ..schema import TableSpec 

from .elements import Dimension, DimensionElement, SkyPixDimension 

from .graph import DimensionGraph 

from .schema import makeOverlapTableSpec, OVERLAP_TABLE_NAME_PATTERN 

from .config import processElementsConfig, processSkyPixConfig, DimensionConfig 

from .packer import DimensionPackerFactory 

 

38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never trueif TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

from .coordinate import ExpandedDataCoordinate 

from .packer import DimensionPacker 

 

 

@immutable 

class DimensionUniverse(DimensionGraph): 

"""A special `DimensionGraph` that constructs and manages a complete set of 

compatible dimensions. 

 

`DimensionUniverse` is not a class-level singleton, but all instances are 

tracked in a singleton map keyed by the version number in the configuration 

they were loaded from. Because these universes are solely responsible for 

constructing `DimensionElement` instances, these are also indirectly 

tracked by that singleton as well. 

 

Parameters 

---------- 

config : `Config`, optional 

Configuration describing the dimensions and their relationships. If 

not provided, default configuration (from 

``daf_butler/config/dimensions.yaml``) wil be loaded. 

""" 

 

_instances = {} 

"""Singleton dictionary of all instances, keyed by version. 

 

For internal use only. 

""" 

 

def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse: 

# Normalize the config and apply defaults. 

config = DimensionConfig(config) 

 

# First see if an equivalent instance already exists. 

version = config["version"] 

self = cls._instances.get(version) 

if self is not None: 

return self 

 

# Create the universe instance and add core attributes. 

# We don't want any of what DimensionGraph.__new__ does, so we just go 

# straight to object.__new__. The C++ side of my brain is offended by 

# this, but I think it's the right approach in Python, where we don't 

# have the option of having multiple constructors with different roles. 

self = object.__new__(cls) 

self.universe = self 

self._cache = {} 

self.dimensions = NamedValueSet() 

self.elements = NamedValueSet() 

 

# Read the skypix dimensions from config. 

skyPixDimensions, self.commonSkyPix = processSkyPixConfig(config["skypix"]) 

# Add the skypix dimensions to the universe after sorting 

# lexicographically (no topological sort because skypix dimensions 

# never have any dependencies). 

for name in sorted(skyPixDimensions): 

skyPixDimensions[name]._finish(self) 

 

# Read the other dimension elements from config. 

elementsToDo = processElementsConfig(config["elements"]) 

# Add elements to the universe in topological order by identifying at 

# each outer iteration which elements have already had all of their 

# dependencies added. 

while elementsToDo: 

unblocked = [name for name, element in elementsToDo.items() 

if element._directDependencyNames.isdisjoint(elementsToDo.keys())] 

unblocked.sort() # Break ties lexicographically. 

if not unblocked: 

raise RuntimeError(f"Cycle detected in dimension elements: {elementsToDo.keys()}.") 

for name in unblocked: 

# Finish initialization of the element with steps that 

# depend on those steps already having been run for all 

# dependencies. 

# This includes adding the element to self.elements and 

# (if appropriate) self.dimensions. 

elementsToDo.pop(name)._finish(self) 

 

# Add attributes for special subsets of the graph. 

self.empty = DimensionGraph(self, (), conform=False) 

self._finish() 

 

# Set up factories for dataId packers as defined by config. 

self._packers = {} 

for name, subconfig in config.get("packers", {}).items(): 

self._packers[name] = DimensionPackerFactory.fromConfig(universe=self, config=subconfig) 

 

# Use the version number from the config as a key in the singleton 

# dict containing all instances; that will let us transfer dimension 

# objects between processes using pickle without actually going 

# through real initialization, as long as a universe with the same 

# version has already been constructed in the receiving process. 

self._version = version 

cls._instances[self._version] = self 

return self 

 

def __repr__(self) -> str: 

return f"DimensionUniverse({self})" 

 

def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

"""Construct a `DimensionGraph` from a possibly-heterogenous iterable 

of `Dimension` instances and string names thereof. 

 

Constructing `DimensionGraph` directly from names or dimension 

instances is slightly more efficient when it is known in advance that 

the iterable is not heterogenous. 

 

Parameters 

---------- 

iterable: iterable of `Dimension` or `str` 

Dimensions that must be included in the returned graph (their 

dependencies will be as well). 

 

Returns 

------- 

graph : `DimensionGraph` 

A `DimensionGraph` instance containing all given dimensions. 

""" 

names = set() 

for item in iterable: 

try: 

names.add(item.name) 

except AttributeError: 

names.add(item) 

return DimensionGraph(universe=self, names=names) 

 

def sorted(self, elements: Iterable[DimensionElement], *, reverse=False) -> List[DimensionElement]: 

"""Return a sorted version of the given iterable of dimension elements. 

 

The universe's sort order is topological (an element's dependencies 

precede it), starting with skypix dimensions (which never have 

dependencies) and then sorting lexicographically to break ties. 

 

Parameters 

---------- 

elements : iterable of `DimensionElement`. 

Elements to be sorted. 

reverse : `bool`, optional 

If `True`, sort in the opposite order. 

 

Returns 

------- 

sorted : `list` of `DimensionElement` 

A sorted list containing the same elements that were given. 

""" 

s = set(elements) 

result = [element for element in self.elements if element in s or element.name in s] 

if reverse: 

result.reverse() 

return result 

 

def makeSchemaSpec(self) -> Dict[str, TableSpec]: 

"""Create a database-agnostic schema specification for all dimensions. 

 

Returns 

------- 

spec : `dict` 

Mapping from `str` logical table name to `TableSpec`. Callers 

may use other names for actual database tables, but should use 

the keys in this dictionary when calling `setupDimensionStorage`. 

""" 

result = {} 

for element in self.elements: 

if element.viewOf is not None: 

continue 

tableSpec = element.makeTableSpec() 

if tableSpec is not None: 

result[element.name] = tableSpec 

if element.spatial: 

tableName = OVERLAP_TABLE_NAME_PATTERN.format(element.name, self.commonSkyPix.name) 

tableSpec = makeOverlapTableSpec(element, self.commonSkyPix) 

result[tableName] = tableSpec 

return result 

 

def makePacker(self, name: str, dataId: ExpandedDataCoordinate) -> DimensionPacker: 

"""Construct a `DimensionPacker` that can pack data ID dictionaries 

into unique integers. 

 

Parameters 

---------- 

name : `str` 

Name of the packer, matching a key in the "packers" section of the 

dimension configuration. 

dataId : `ExpandedDataCoordinate` 

Fully-expanded data ID that identfies the at least the "fixed" 

dimensions of the packer (i.e. those that are assumed/given, 

setting the space over which packed integer IDs are unique). 

""" 

return self._packers[name](dataId) 

 

@classmethod 

def _unpickle(cls, version: bytes) -> DimensionUniverse: 

"""Callable used for unpickling. 

 

For internal use only. 

""" 

try: 

return cls._instances[version] 

except KeyError as err: 

raise pickle.UnpicklingError( 

f"DimensionUniverse with version '{version}' " 

f"not found. Note that DimensionUniverse objects are not " 

f"truly serialized; when using pickle to transfer them " 

f"between processes, an equivalent instance with the same " 

f"version must already exist in the receiving process." 

) from err 

 

def __reduce__(self) -> tuple: 

return (self._unpickle, (self._version,)) 

 

# Class attributes below are shadowed by instance attributes, and are 

# present just to hold the docstrings for those instance attributes. 

 

empty: DimensionGraph 

"""The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

""" 

 

commonSkyPix: SkyPixDimension 

"""The special skypix dimension that is used to relate all other spatial 

dimensions in the `Registry` database (`SkyPixDimension`). 

"""