Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 24%

75 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-23 02:27 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26from typing import Iterable, Iterator, Optional, Union 

27 

28from lsst.resources import ResourcePath, ResourcePathExpression 

29 

30from .. import ddl 

31from .._topology import TopologicalSpace 

32from ..config import Config, ConfigSubset 

33from ._database import ( 

34 DatabaseDimensionElementConstructionVisitor, 

35 DatabaseTopologicalFamilyConstructionVisitor, 

36) 

37from ._governor import GovernorDimensionConstructionVisitor 

38from ._packer import DimensionPackerConstructionVisitor 

39from ._skypix import SkyPixConstructionVisitor 

40from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

41 

42# The default namespace to use on older dimension config files that only 

43# have a version. 

44_DEFAULT_NAMESPACE = "daf_butler" 

45 

46 

47class DimensionConfig(ConfigSubset): 

48 """Configuration that defines a `DimensionUniverse`. 

49 

50 The configuration tree for dimensions is a (nested) dictionary 

51 with five top-level entries: 

52 

53 - version: an integer version number, used as keys in a singleton registry 

54 of all `DimensionUniverse` instances; 

55 

56 - namespace: a string to be associated with the version in the singleton 

57 registry of all `DimensionUnivers` instances; 

58 

59 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

60 along with a special "common" key whose value is the name of a skypix 

61 dimension that is used to relate all other spatial dimensions in the 

62 `Registry` database; 

63 

64 - elements: a nested dictionary whose entries each define 

65 `StandardDimension` or `StandardDimensionCombination`. 

66 

67 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

68 with dictionary values that each define a `StandardTopologicalFamily`. 

69 

70 - packers: a nested dictionary whose entries define factories for a 

71 `DimensionPacker` instance. 

72 

73 See the documentation for the linked classes above for more information 

74 on the configuration syntax. 

75 

76 Parameters 

77 ---------- 

78 other : `Config` or `str` or `dict`, optional 

79 Argument specifying the configuration information as understood 

80 by `Config`. If `None` is passed then defaults are loaded from 

81 "dimensions.yaml", otherwise defaults are not loaded. 

82 validate : `bool`, optional 

83 If `True` required keys will be checked to ensure configuration 

84 consistency. 

85 searchPaths : `list` or `tuple`, optional 

86 Explicit additional paths to search for defaults. They should 

87 be supplied in priority order. These paths have higher priority 

88 than those read from the environment in 

89 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

90 the local file system or URIs, `lsst.resources.ResourcePath`. 

91 """ 

92 

93 requiredKeys = ("version", "elements", "skypix") 

94 defaultConfigFile = "dimensions.yaml" 

95 

96 def __init__( 

97 self, 

98 other: Union[Config, ResourcePathExpression, None] = None, 

99 validate: bool = True, 

100 searchPaths: Optional[Iterable[ResourcePathExpression]] = None, 

101 ): 

102 # if argument is not None then do not load/merge defaults 

103 mergeDefaults = other is None 

104 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

105 

106 def _updateWithConfigsFromPath( 

107 self, searchPaths: Iterable[ResourcePathExpression], configFile: str 

108 ) -> None: 

109 """Search the supplied paths reading config from first found. 

110 

111 Raises 

112 ------ 

113 FileNotFoundError 

114 Raised if config file is not found in any of given locations. 

115 

116 Notes 

117 ----- 

118 This method overrides base class method with different behavior. 

119 Instead of merging all found files into a single configuration it 

120 finds first matching file and reads it. 

121 """ 

122 uri = ResourcePath(configFile) 

123 if uri.isabs() and uri.exists(): 

124 # Assume this resource exists 

125 self._updateWithOtherConfigFile(configFile) 

126 self.filesRead.append(configFile) 

127 else: 

128 for pathDir in searchPaths: 

129 if isinstance(pathDir, (str, ResourcePath)): 

130 pathDir = ResourcePath(pathDir, forceDirectory=True) 

131 file = pathDir.join(configFile) 

132 if file.exists(): 

133 self.filesRead.append(file) 

134 self._updateWithOtherConfigFile(file) 

135 break 

136 else: 

137 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

138 else: 

139 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

140 

141 def _updateWithOtherConfigFile(self, file: Union[ResourcePath, str]) -> None: 

142 """Override for base class method. 

143 

144 Parameters 

145 ---------- 

146 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

147 Entity that can be converted to a `ConfigSubset`. 

148 """ 

149 # Use this class to read the defaults so that subsetting can happen 

150 # correctly. 

151 externalConfig = type(self)(file, validate=False) 

152 self.update(externalConfig) 

153 

154 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

155 """Process the 'skypix' section of the configuration. 

156 

157 Yields a construction visitor for each `SkyPixSystem`. 

158 

159 Yields 

160 ------ 

161 visitor : `DimensionConstructionVisitor` 

162 Object that adds a skypix system and its dimensions to an 

163 under-construction `DimensionUniverse`. 

164 """ 

165 config = self["skypix"] 

166 systemNames = set(config.keys()) 

167 systemNames.remove("common") 

168 for systemName in sorted(systemNames): 

169 subconfig = config[systemName] 

170 pixelizationClassName = subconfig["class"] 

171 maxLevel = subconfig.get("max_level", 24) 

172 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

173 

174 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

175 """Process the 'elements' section of the configuration. 

176 

177 Yields a construction visitor for each `StandardDimension` or 

178 `StandardDimensionCombination`. 

179 

180 Yields 

181 ------ 

182 visitor : `DimensionConstructionVisitor` 

183 Object that adds a `StandardDimension` or 

184 `StandardDimensionCombination` to an under-construction 

185 `DimensionUniverse`. 

186 """ 

187 for name, subconfig in self["elements"].items(): 

188 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

189 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

190 if uniqueKeys: 

191 uniqueKeys[0].primaryKey = True 

192 if subconfig.get("governor", False): 

193 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

194 if not unsupported.isdisjoint(subconfig.keys()): 

195 raise RuntimeError( 

196 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

197 ) 

198 if not subconfig.get("cached", True): 

199 raise RuntimeError(f"Governor dimension {name} is always cached.") 

200 yield GovernorDimensionConstructionVisitor( 

201 name=name, 

202 storage=subconfig["storage"], 

203 metadata=metadata, 

204 uniqueKeys=uniqueKeys, 

205 ) 

206 else: 

207 yield DatabaseDimensionElementConstructionVisitor( 

208 name=name, 

209 storage=subconfig["storage"], 

210 required=set(subconfig.get("requires", ())), 

211 implied=set(subconfig.get("implies", ())), 

212 metadata=metadata, 

213 alwaysJoin=subconfig.get("always_join", False), 

214 uniqueKeys=uniqueKeys, 

215 ) 

216 

217 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

218 """Process the 'topology' section of the configuration. 

219 

220 Yields a construction visitor for each `StandardTopologicalFamily`. 

221 

222 Yields 

223 ------ 

224 visitor : `DimensionConstructionVisitor` 

225 Object that adds a `StandardTopologicalFamily` to an 

226 under-construction `DimensionUniverse` and updates its member 

227 `DimensionElement` instances. 

228 """ 

229 for spaceName, subconfig in self.get("topology", {}).items(): 

230 space = TopologicalSpace.__members__[spaceName.upper()] 

231 for name, members in subconfig.items(): 

232 yield DatabaseTopologicalFamilyConstructionVisitor( 

233 name=name, 

234 space=space, 

235 members=members, 

236 ) 

237 

238 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

239 """Process the 'packers' section of the configuration. 

240 

241 Yields construction visitors for each `DimensionPackerFactory`. 

242 

243 Yields 

244 ------ 

245 visitor : `DimensionConstructionVisitor` 

246 Object that adds a `DinmensionPackerFactory` to an 

247 under-construction `DimensionUniverse`. 

248 """ 

249 for name, subconfig in self["packers"].items(): 

250 yield DimensionPackerConstructionVisitor( 

251 name=name, 

252 clsName=subconfig["cls"], 

253 fixed=subconfig["fixed"], 

254 dimensions=subconfig["dimensions"], 

255 ) 

256 

257 def makeBuilder(self) -> DimensionConstructionBuilder: 

258 """Construct a `DinmensionConstructionBuilder`. 

259 

260 The builder will reflect this configuration. 

261 

262 Returns 

263 ------- 

264 builder : `DimensionConstructionBuilder` 

265 A builder object populated with all visitors from this 

266 configuration. The `~DimensionConstructionBuilder.finish` method 

267 will not have been called. 

268 """ 

269 builder = DimensionConstructionBuilder( 

270 self["version"], 

271 self["skypix", "common"], 

272 self, 

273 namespace=self.get("namespace", _DEFAULT_NAMESPACE), 

274 ) 

275 builder.update(self._extractSkyPixVisitors()) 

276 builder.update(self._extractElementVisitors()) 

277 builder.update(self._extractTopologyVisitors()) 

278 builder.update(self._extractPackerVisitors()) 

279 return builder