Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26from typing import Iterator, Iterable, Optional, Union 

27 

28from ..config import Config, ConfigSubset 

29from .. import ddl 

30from .._butlerUri import ButlerURI 

31from .._topology import TopologicalSpace 

32from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

33from ._governor import GovernorDimensionConstructionVisitor 

34from ._packer import DimensionPackerConstructionVisitor 

35from ._skypix import SkyPixConstructionVisitor 

36from ._database import ( 

37 DatabaseDimensionElementConstructionVisitor, 

38 DatabaseTopologicalFamilyConstructionVisitor, 

39) 

40 

41 

42class DimensionConfig(ConfigSubset): 

43 """Configuration that defines a `DimensionUniverse`. 

44 

45 The configuration tree for dimensions is a (nested) dictionary 

46 with five top-level entries: 

47 

48 - version: an integer version number, used as keys in a singleton registry 

49 of all `DimensionUniverse` instances; 

50 

51 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

52 along with a special "common" key whose value is the name of a skypix 

53 dimension that is used to relate all other spatial dimensions in the 

54 `Registry` database; 

55 

56 - elements: a nested dictionary whose entries each define 

57 `StandardDimension` or `StandardDimensionCombination`. 

58 

59 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

60 with dictionary values that each define a `StandardTopologicalFamily`. 

61 

62 - packers: a nested dictionary whose entries define factories for a 

63 `DimensionPacker` instance. 

64 

65 See the documentation for the linked classes above for more information 

66 on the configuration syntax. 

67 

68 Parameters 

69 ---------- 

70 other : `Config` or `str` or `dict`, optional 

71 Argument specifying the configuration information as understood 

72 by `Config`. If `None` is passed then defaults are loaded from 

73 "dimensions.yaml", otherwise defaults are not loaded. 

74 validate : `bool`, optional 

75 If `True` required keys will be checked to ensure configuration 

76 consistency. 

77 searchPaths : `list` or `tuple`, optional 

78 Explicit additional paths to search for defaults. They should 

79 be supplied in priority order. These paths have higher priority 

80 than those read from the environment in 

81 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

82 the local file system or URIs, `ButlerURI`. 

83 """ 

84 

85 requiredKeys = ("version", "elements", "skypix") 

86 defaultConfigFile = "dimensions.yaml" 

87 

88 def __init__(self, other: Union[Config, ButlerURI, str, None] = None, validate: bool = True, 

89 searchPaths: Optional[Iterable[Union[str, ButlerURI]]] = None): 

90 # if argument is not None then do not load/merge defaults 

91 mergeDefaults = other is None 

92 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, 

93 searchPaths=searchPaths) 

94 

95 def _updateWithConfigsFromPath(self, searchPaths: Iterable[Union[str, ButlerURI]], 

96 configFile: str) -> None: 

97 """Search the supplied paths reading config from first found. 

98 

99 Raises 

100 ------ 

101 FileNotFoundError 

102 Raised if config file is not found in any of given locations. 

103 

104 Notes 

105 ----- 

106 This method overrides base class method with different behavior. 

107 Instead of merging all found files into a single configuration it 

108 finds first matching file and reads it. 

109 """ 

110 uri = ButlerURI(configFile) 

111 if uri.isabs() and uri.exists(): 

112 # Assume this resource exists 

113 self._updateWithOtherConfigFile(configFile) 

114 self.filesRead.append(configFile) 

115 else: 

116 for pathDir in searchPaths: 

117 if isinstance(pathDir, (str, ButlerURI)): 

118 pathDir = ButlerURI(pathDir, forceDirectory=True) 

119 file = pathDir.join(configFile) 

120 if file.exists(): 

121 self.filesRead.append(file) 

122 self._updateWithOtherConfigFile(file) 

123 break 

124 else: 

125 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

126 else: 

127 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

128 

129 def _updateWithOtherConfigFile(self, file: Union[ButlerURI, str]) -> None: 

130 """Override for base class method. 

131 

132 Parameters 

133 ---------- 

134 file : `Config`, `str`, `ButlerURI`, or `dict` 

135 Entity that can be converted to a `ConfigSubset`. 

136 """ 

137 # Use this class to read the defaults so that subsetting can happen 

138 # correctly. 

139 externalConfig = type(self)(file, validate=False) 

140 self.update(externalConfig) 

141 

142 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

143 """Process the 'skypix' section of the configuration. 

144 

145 Yields a construction visitor for each `SkyPixSystem`. 

146 

147 Yields 

148 ------ 

149 visitor : `DimensionConstructionVisitor` 

150 Object that adds a skypix system and its dimensions to an 

151 under-construction `DimensionUniverse`. 

152 """ 

153 config = self["skypix"] 

154 systemNames = set(config.keys()) 

155 systemNames.remove("common") 

156 for systemName in sorted(systemNames): 

157 subconfig = config[systemName] 

158 pixelizationClassName = subconfig["class"] 

159 maxLevel = subconfig.get("max_level", 24) 

160 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

161 

162 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

163 """Process the 'elements' section of the configuration. 

164 

165 Yields a construction visitor for each `StandardDimension` or 

166 `StandardDimensionCombination`. 

167 

168 Yields 

169 ------ 

170 visitor : `DimensionConstructionVisitor` 

171 Object that adds a `StandardDimension` or 

172 `StandardDimensionCombination` to an under-construction 

173 `DimensionUniverse`. 

174 """ 

175 for name, subconfig in self["elements"].items(): 

176 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

177 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

178 if uniqueKeys: 

179 uniqueKeys[0].primaryKey = True 

180 if subconfig.get("governor", False): 

181 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

182 if not unsupported.isdisjoint(subconfig.keys()): 

183 raise RuntimeError( 

184 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

185 ) 

186 if not subconfig.get("cached", True): 

187 raise RuntimeError( 

188 f"Governor dimension {name} is always cached." 

189 ) 

190 yield GovernorDimensionConstructionVisitor( 

191 name=name, 

192 storage=subconfig["storage"], 

193 metadata=metadata, 

194 uniqueKeys=uniqueKeys, 

195 ) 

196 else: 

197 yield DatabaseDimensionElementConstructionVisitor( 

198 name=name, 

199 storage=subconfig["storage"], 

200 required=set(subconfig.get("requires", ())), 

201 implied=set(subconfig.get("implies", ())), 

202 metadata=metadata, 

203 alwaysJoin=subconfig.get("always_join", False), 

204 uniqueKeys=uniqueKeys, 

205 ) 

206 

207 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

208 """Process the 'topology' section of the configuration. 

209 

210 Yields a construction visitor for each `StandardTopologicalFamily`. 

211 

212 Yields 

213 ------ 

214 visitor : `DimensionConstructionVisitor` 

215 Object that adds a `StandardTopologicalFamily` to an 

216 under-construction `DimensionUniverse` and updates its member 

217 `DimensionElement` instances. 

218 """ 

219 for spaceName, subconfig in self.get("topology", {}).items(): 

220 space = TopologicalSpace.__members__[spaceName.upper()] 

221 for name, members in subconfig.items(): 

222 yield DatabaseTopologicalFamilyConstructionVisitor( 

223 name=name, 

224 space=space, 

225 members=members, 

226 ) 

227 

228 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

229 """Process the 'packers' section of the configuration. 

230 

231 Yields construction visitors for each `DimensionPackerFactory`. 

232 

233 Yields 

234 ------ 

235 visitor : `DimensionConstructionVisitor` 

236 Object that adds a `DinmensionPackerFactory` to an 

237 under-construction `DimensionUniverse`. 

238 """ 

239 for name, subconfig in self["packers"].items(): 

240 yield DimensionPackerConstructionVisitor( 

241 name=name, 

242 clsName=subconfig["cls"], 

243 fixed=subconfig["fixed"], 

244 dimensions=subconfig["dimensions"], 

245 ) 

246 

247 def makeBuilder(self) -> DimensionConstructionBuilder: 

248 """Construct a `DinmensionConstructionBuilder`. 

249 

250 The builder will reflect this configuration. 

251 

252 Returns 

253 ------- 

254 builder : `DimensionConstructionBuilder` 

255 A builder object populated with all visitors from this 

256 configuration. The `~DimensionConstructionBuilder.finish` method 

257 will not have been called. 

258 """ 

259 builder = DimensionConstructionBuilder(self["version"], self["skypix", "common"], self) 

260 builder.update(self._extractSkyPixVisitors()) 

261 builder.update(self._extractElementVisitors()) 

262 builder.update(self._extractTopologyVisitors()) 

263 builder.update(self._extractPackerVisitors()) 

264 return builder