Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

74 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26from typing import Iterable, Iterator, Optional, Union 

27 

28from .. import ddl 

29from .._butlerUri import ButlerURI 

30from .._topology import TopologicalSpace 

31from ..config import Config, ConfigSubset 

32from ._database import ( 

33 DatabaseDimensionElementConstructionVisitor, 

34 DatabaseTopologicalFamilyConstructionVisitor, 

35) 

36from ._governor import GovernorDimensionConstructionVisitor 

37from ._packer import DimensionPackerConstructionVisitor 

38from ._skypix import SkyPixConstructionVisitor 

39from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

40 

41 

42class DimensionConfig(ConfigSubset): 

43 """Configuration that defines a `DimensionUniverse`. 

44 

45 The configuration tree for dimensions is a (nested) dictionary 

46 with five top-level entries: 

47 

48 - version: an integer version number, used as keys in a singleton registry 

49 of all `DimensionUniverse` instances; 

50 

51 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

52 along with a special "common" key whose value is the name of a skypix 

53 dimension that is used to relate all other spatial dimensions in the 

54 `Registry` database; 

55 

56 - elements: a nested dictionary whose entries each define 

57 `StandardDimension` or `StandardDimensionCombination`. 

58 

59 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

60 with dictionary values that each define a `StandardTopologicalFamily`. 

61 

62 - packers: a nested dictionary whose entries define factories for a 

63 `DimensionPacker` instance. 

64 

65 See the documentation for the linked classes above for more information 

66 on the configuration syntax. 

67 

68 Parameters 

69 ---------- 

70 other : `Config` or `str` or `dict`, optional 

71 Argument specifying the configuration information as understood 

72 by `Config`. If `None` is passed then defaults are loaded from 

73 "dimensions.yaml", otherwise defaults are not loaded. 

74 validate : `bool`, optional 

75 If `True` required keys will be checked to ensure configuration 

76 consistency. 

77 searchPaths : `list` or `tuple`, optional 

78 Explicit additional paths to search for defaults. They should 

79 be supplied in priority order. These paths have higher priority 

80 than those read from the environment in 

81 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

82 the local file system or URIs, `ButlerURI`. 

83 """ 

84 

85 requiredKeys = ("version", "elements", "skypix") 

86 defaultConfigFile = "dimensions.yaml" 

87 

88 def __init__( 

89 self, 

90 other: Union[Config, ButlerURI, str, None] = None, 

91 validate: bool = True, 

92 searchPaths: Optional[Iterable[Union[str, ButlerURI]]] = None, 

93 ): 

94 # if argument is not None then do not load/merge defaults 

95 mergeDefaults = other is None 

96 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

97 

98 def _updateWithConfigsFromPath( 

99 self, searchPaths: Iterable[Union[str, ButlerURI]], configFile: str 

100 ) -> None: 

101 """Search the supplied paths reading config from first found. 

102 

103 Raises 

104 ------ 

105 FileNotFoundError 

106 Raised if config file is not found in any of given locations. 

107 

108 Notes 

109 ----- 

110 This method overrides base class method with different behavior. 

111 Instead of merging all found files into a single configuration it 

112 finds first matching file and reads it. 

113 """ 

114 uri = ButlerURI(configFile) 

115 if uri.isabs() and uri.exists(): 

116 # Assume this resource exists 

117 self._updateWithOtherConfigFile(configFile) 

118 self.filesRead.append(configFile) 

119 else: 

120 for pathDir in searchPaths: 

121 if isinstance(pathDir, (str, ButlerURI)): 

122 pathDir = ButlerURI(pathDir, forceDirectory=True) 

123 file = pathDir.join(configFile) 

124 if file.exists(): 

125 self.filesRead.append(file) 

126 self._updateWithOtherConfigFile(file) 

127 break 

128 else: 

129 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

130 else: 

131 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

132 

133 def _updateWithOtherConfigFile(self, file: Union[ButlerURI, str]) -> None: 

134 """Override for base class method. 

135 

136 Parameters 

137 ---------- 

138 file : `Config`, `str`, `ButlerURI`, or `dict` 

139 Entity that can be converted to a `ConfigSubset`. 

140 """ 

141 # Use this class to read the defaults so that subsetting can happen 

142 # correctly. 

143 externalConfig = type(self)(file, validate=False) 

144 self.update(externalConfig) 

145 

146 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

147 """Process the 'skypix' section of the configuration. 

148 

149 Yields a construction visitor for each `SkyPixSystem`. 

150 

151 Yields 

152 ------ 

153 visitor : `DimensionConstructionVisitor` 

154 Object that adds a skypix system and its dimensions to an 

155 under-construction `DimensionUniverse`. 

156 """ 

157 config = self["skypix"] 

158 systemNames = set(config.keys()) 

159 systemNames.remove("common") 

160 for systemName in sorted(systemNames): 

161 subconfig = config[systemName] 

162 pixelizationClassName = subconfig["class"] 

163 maxLevel = subconfig.get("max_level", 24) 

164 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

165 

166 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

167 """Process the 'elements' section of the configuration. 

168 

169 Yields a construction visitor for each `StandardDimension` or 

170 `StandardDimensionCombination`. 

171 

172 Yields 

173 ------ 

174 visitor : `DimensionConstructionVisitor` 

175 Object that adds a `StandardDimension` or 

176 `StandardDimensionCombination` to an under-construction 

177 `DimensionUniverse`. 

178 """ 

179 for name, subconfig in self["elements"].items(): 

180 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

181 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

182 if uniqueKeys: 

183 uniqueKeys[0].primaryKey = True 

184 if subconfig.get("governor", False): 

185 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

186 if not unsupported.isdisjoint(subconfig.keys()): 

187 raise RuntimeError( 

188 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

189 ) 

190 if not subconfig.get("cached", True): 

191 raise RuntimeError(f"Governor dimension {name} is always cached.") 

192 yield GovernorDimensionConstructionVisitor( 

193 name=name, 

194 storage=subconfig["storage"], 

195 metadata=metadata, 

196 uniqueKeys=uniqueKeys, 

197 ) 

198 else: 

199 yield DatabaseDimensionElementConstructionVisitor( 

200 name=name, 

201 storage=subconfig["storage"], 

202 required=set(subconfig.get("requires", ())), 

203 implied=set(subconfig.get("implies", ())), 

204 metadata=metadata, 

205 alwaysJoin=subconfig.get("always_join", False), 

206 uniqueKeys=uniqueKeys, 

207 ) 

208 

209 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

210 """Process the 'topology' section of the configuration. 

211 

212 Yields a construction visitor for each `StandardTopologicalFamily`. 

213 

214 Yields 

215 ------ 

216 visitor : `DimensionConstructionVisitor` 

217 Object that adds a `StandardTopologicalFamily` to an 

218 under-construction `DimensionUniverse` and updates its member 

219 `DimensionElement` instances. 

220 """ 

221 for spaceName, subconfig in self.get("topology", {}).items(): 

222 space = TopologicalSpace.__members__[spaceName.upper()] 

223 for name, members in subconfig.items(): 

224 yield DatabaseTopologicalFamilyConstructionVisitor( 

225 name=name, 

226 space=space, 

227 members=members, 

228 ) 

229 

230 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

231 """Process the 'packers' section of the configuration. 

232 

233 Yields construction visitors for each `DimensionPackerFactory`. 

234 

235 Yields 

236 ------ 

237 visitor : `DimensionConstructionVisitor` 

238 Object that adds a `DinmensionPackerFactory` to an 

239 under-construction `DimensionUniverse`. 

240 """ 

241 for name, subconfig in self["packers"].items(): 

242 yield DimensionPackerConstructionVisitor( 

243 name=name, 

244 clsName=subconfig["cls"], 

245 fixed=subconfig["fixed"], 

246 dimensions=subconfig["dimensions"], 

247 ) 

248 

249 def makeBuilder(self) -> DimensionConstructionBuilder: 

250 """Construct a `DinmensionConstructionBuilder`. 

251 

252 The builder will reflect this configuration. 

253 

254 Returns 

255 ------- 

256 builder : `DimensionConstructionBuilder` 

257 A builder object populated with all visitors from this 

258 configuration. The `~DimensionConstructionBuilder.finish` method 

259 will not have been called. 

260 """ 

261 builder = DimensionConstructionBuilder(self["version"], self["skypix", "common"], self) 

262 builder.update(self._extractSkyPixVisitors()) 

263 builder.update(self._extractElementVisitors()) 

264 builder.update(self._extractTopologyVisitors()) 

265 builder.update(self._extractPackerVisitors()) 

266 return builder