Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

74 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26from typing import Iterable, Iterator, Optional, Union 

27 

28from lsst.resources import ResourcePath, ResourcePathExpression 

29 

30from .. import ddl 

31from .._topology import TopologicalSpace 

32from ..config import Config, ConfigSubset 

33from ._database import ( 

34 DatabaseDimensionElementConstructionVisitor, 

35 DatabaseTopologicalFamilyConstructionVisitor, 

36) 

37from ._governor import GovernorDimensionConstructionVisitor 

38from ._packer import DimensionPackerConstructionVisitor 

39from ._skypix import SkyPixConstructionVisitor 

40from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

41 

42 

43class DimensionConfig(ConfigSubset): 

44 """Configuration that defines a `DimensionUniverse`. 

45 

46 The configuration tree for dimensions is a (nested) dictionary 

47 with five top-level entries: 

48 

49 - version: an integer version number, used as keys in a singleton registry 

50 of all `DimensionUniverse` instances; 

51 

52 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

53 along with a special "common" key whose value is the name of a skypix 

54 dimension that is used to relate all other spatial dimensions in the 

55 `Registry` database; 

56 

57 - elements: a nested dictionary whose entries each define 

58 `StandardDimension` or `StandardDimensionCombination`. 

59 

60 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

61 with dictionary values that each define a `StandardTopologicalFamily`. 

62 

63 - packers: a nested dictionary whose entries define factories for a 

64 `DimensionPacker` instance. 

65 

66 See the documentation for the linked classes above for more information 

67 on the configuration syntax. 

68 

69 Parameters 

70 ---------- 

71 other : `Config` or `str` or `dict`, optional 

72 Argument specifying the configuration information as understood 

73 by `Config`. If `None` is passed then defaults are loaded from 

74 "dimensions.yaml", otherwise defaults are not loaded. 

75 validate : `bool`, optional 

76 If `True` required keys will be checked to ensure configuration 

77 consistency. 

78 searchPaths : `list` or `tuple`, optional 

79 Explicit additional paths to search for defaults. They should 

80 be supplied in priority order. These paths have higher priority 

81 than those read from the environment in 

82 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

83 the local file system or URIs, `lsst.resources.ResourcePath`. 

84 """ 

85 

86 requiredKeys = ("version", "elements", "skypix") 

87 defaultConfigFile = "dimensions.yaml" 

88 

89 def __init__( 

90 self, 

91 other: Union[Config, ResourcePathExpression, None] = None, 

92 validate: bool = True, 

93 searchPaths: Optional[Iterable[ResourcePathExpression]] = None, 

94 ): 

95 # if argument is not None then do not load/merge defaults 

96 mergeDefaults = other is None 

97 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

98 

99 def _updateWithConfigsFromPath( 

100 self, searchPaths: Iterable[ResourcePathExpression], configFile: str 

101 ) -> None: 

102 """Search the supplied paths reading config from first found. 

103 

104 Raises 

105 ------ 

106 FileNotFoundError 

107 Raised if config file is not found in any of given locations. 

108 

109 Notes 

110 ----- 

111 This method overrides base class method with different behavior. 

112 Instead of merging all found files into a single configuration it 

113 finds first matching file and reads it. 

114 """ 

115 uri = ResourcePath(configFile) 

116 if uri.isabs() and uri.exists(): 

117 # Assume this resource exists 

118 self._updateWithOtherConfigFile(configFile) 

119 self.filesRead.append(configFile) 

120 else: 

121 for pathDir in searchPaths: 

122 if isinstance(pathDir, (str, ResourcePath)): 

123 pathDir = ResourcePath(pathDir, forceDirectory=True) 

124 file = pathDir.join(configFile) 

125 if file.exists(): 

126 self.filesRead.append(file) 

127 self._updateWithOtherConfigFile(file) 

128 break 

129 else: 

130 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

131 else: 

132 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

133 

134 def _updateWithOtherConfigFile(self, file: Union[ResourcePath, str]) -> None: 

135 """Override for base class method. 

136 

137 Parameters 

138 ---------- 

139 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

140 Entity that can be converted to a `ConfigSubset`. 

141 """ 

142 # Use this class to read the defaults so that subsetting can happen 

143 # correctly. 

144 externalConfig = type(self)(file, validate=False) 

145 self.update(externalConfig) 

146 

147 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

148 """Process the 'skypix' section of the configuration. 

149 

150 Yields a construction visitor for each `SkyPixSystem`. 

151 

152 Yields 

153 ------ 

154 visitor : `DimensionConstructionVisitor` 

155 Object that adds a skypix system and its dimensions to an 

156 under-construction `DimensionUniverse`. 

157 """ 

158 config = self["skypix"] 

159 systemNames = set(config.keys()) 

160 systemNames.remove("common") 

161 for systemName in sorted(systemNames): 

162 subconfig = config[systemName] 

163 pixelizationClassName = subconfig["class"] 

164 maxLevel = subconfig.get("max_level", 24) 

165 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

166 

167 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

168 """Process the 'elements' section of the configuration. 

169 

170 Yields a construction visitor for each `StandardDimension` or 

171 `StandardDimensionCombination`. 

172 

173 Yields 

174 ------ 

175 visitor : `DimensionConstructionVisitor` 

176 Object that adds a `StandardDimension` or 

177 `StandardDimensionCombination` to an under-construction 

178 `DimensionUniverse`. 

179 """ 

180 for name, subconfig in self["elements"].items(): 

181 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

182 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

183 if uniqueKeys: 

184 uniqueKeys[0].primaryKey = True 

185 if subconfig.get("governor", False): 

186 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

187 if not unsupported.isdisjoint(subconfig.keys()): 

188 raise RuntimeError( 

189 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

190 ) 

191 if not subconfig.get("cached", True): 

192 raise RuntimeError(f"Governor dimension {name} is always cached.") 

193 yield GovernorDimensionConstructionVisitor( 

194 name=name, 

195 storage=subconfig["storage"], 

196 metadata=metadata, 

197 uniqueKeys=uniqueKeys, 

198 ) 

199 else: 

200 yield DatabaseDimensionElementConstructionVisitor( 

201 name=name, 

202 storage=subconfig["storage"], 

203 required=set(subconfig.get("requires", ())), 

204 implied=set(subconfig.get("implies", ())), 

205 metadata=metadata, 

206 alwaysJoin=subconfig.get("always_join", False), 

207 uniqueKeys=uniqueKeys, 

208 ) 

209 

210 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

211 """Process the 'topology' section of the configuration. 

212 

213 Yields a construction visitor for each `StandardTopologicalFamily`. 

214 

215 Yields 

216 ------ 

217 visitor : `DimensionConstructionVisitor` 

218 Object that adds a `StandardTopologicalFamily` to an 

219 under-construction `DimensionUniverse` and updates its member 

220 `DimensionElement` instances. 

221 """ 

222 for spaceName, subconfig in self.get("topology", {}).items(): 

223 space = TopologicalSpace.__members__[spaceName.upper()] 

224 for name, members in subconfig.items(): 

225 yield DatabaseTopologicalFamilyConstructionVisitor( 

226 name=name, 

227 space=space, 

228 members=members, 

229 ) 

230 

231 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

232 """Process the 'packers' section of the configuration. 

233 

234 Yields construction visitors for each `DimensionPackerFactory`. 

235 

236 Yields 

237 ------ 

238 visitor : `DimensionConstructionVisitor` 

239 Object that adds a `DinmensionPackerFactory` to an 

240 under-construction `DimensionUniverse`. 

241 """ 

242 for name, subconfig in self["packers"].items(): 

243 yield DimensionPackerConstructionVisitor( 

244 name=name, 

245 clsName=subconfig["cls"], 

246 fixed=subconfig["fixed"], 

247 dimensions=subconfig["dimensions"], 

248 ) 

249 

250 def makeBuilder(self) -> DimensionConstructionBuilder: 

251 """Construct a `DinmensionConstructionBuilder`. 

252 

253 The builder will reflect this configuration. 

254 

255 Returns 

256 ------- 

257 builder : `DimensionConstructionBuilder` 

258 A builder object populated with all visitors from this 

259 configuration. The `~DimensionConstructionBuilder.finish` method 

260 will not have been called. 

261 """ 

262 builder = DimensionConstructionBuilder(self["version"], self["skypix", "common"], self) 

263 builder.update(self._extractSkyPixVisitors()) 

264 builder.update(self._extractElementVisitors()) 

265 builder.update(self._extractTopologyVisitors()) 

266 builder.update(self._extractPackerVisitors()) 

267 return builder