Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 24%

76 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 02:18 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26from collections.abc import Iterator, Mapping, Sequence 

27from typing import Any 

28 

29from lsst.resources import ResourcePath, ResourcePathExpression 

30 

31from .. import ddl 

32from .._topology import TopologicalSpace 

33from ..config import Config, ConfigSubset 

34from ._database import ( 

35 DatabaseDimensionElementConstructionVisitor, 

36 DatabaseTopologicalFamilyConstructionVisitor, 

37) 

38from ._governor import GovernorDimensionConstructionVisitor 

39from ._packer import DimensionPackerConstructionVisitor 

40from ._skypix import SkyPixConstructionVisitor 

41from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

42 

43# The default namespace to use on older dimension config files that only 

44# have a version. 

45_DEFAULT_NAMESPACE = "daf_butler" 

46 

47 

48class DimensionConfig(ConfigSubset): 

49 """Configuration that defines a `DimensionUniverse`. 

50 

51 The configuration tree for dimensions is a (nested) dictionary 

52 with five top-level entries: 

53 

54 - version: an integer version number, used as keys in a singleton registry 

55 of all `DimensionUniverse` instances; 

56 

57 - namespace: a string to be associated with the version in the singleton 

58 registry of all `DimensionUnivers` instances; 

59 

60 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

61 along with a special "common" key whose value is the name of a skypix 

62 dimension that is used to relate all other spatial dimensions in the 

63 `Registry` database; 

64 

65 - elements: a nested dictionary whose entries each define 

66 `StandardDimension` or `StandardDimensionCombination`. 

67 

68 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

69 with dictionary values that each define a `StandardTopologicalFamily`. 

70 

71 - packers: a nested dictionary whose entries define factories for a 

72 `DimensionPacker` instance. 

73 

74 See the documentation for the linked classes above for more information 

75 on the configuration syntax. 

76 

77 Parameters 

78 ---------- 

79 other : `Config` or `str` or `dict`, optional 

80 Argument specifying the configuration information as understood 

81 by `Config`. If `None` is passed then defaults are loaded from 

82 "dimensions.yaml", otherwise defaults are not loaded. 

83 validate : `bool`, optional 

84 If `True` required keys will be checked to ensure configuration 

85 consistency. 

86 searchPaths : `list` or `tuple`, optional 

87 Explicit additional paths to search for defaults. They should 

88 be supplied in priority order. These paths have higher priority 

89 than those read from the environment in 

90 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

91 the local file system or URIs, `lsst.resources.ResourcePath`. 

92 """ 

93 

94 requiredKeys = ("version", "elements", "skypix") 

95 defaultConfigFile = "dimensions.yaml" 

96 

97 def __init__( 

98 self, 

99 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None, 

100 validate: bool = True, 

101 searchPaths: Sequence[ResourcePathExpression] | None = None, 

102 ): 

103 # if argument is not None then do not load/merge defaults 

104 mergeDefaults = other is None 

105 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

106 

107 def _updateWithConfigsFromPath( 

108 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str 

109 ) -> None: 

110 """Search the supplied paths reading config from first found. 

111 

112 Raises 

113 ------ 

114 FileNotFoundError 

115 Raised if config file is not found in any of given locations. 

116 

117 Notes 

118 ----- 

119 This method overrides base class method with different behavior. 

120 Instead of merging all found files into a single configuration it 

121 finds first matching file and reads it. 

122 """ 

123 uri = ResourcePath(configFile) 

124 if uri.isabs() and uri.exists(): 

125 # Assume this resource exists 

126 self._updateWithOtherConfigFile(configFile) 

127 self.filesRead.append(configFile) 

128 else: 

129 for pathDir in searchPaths: 

130 if isinstance(pathDir, (str, ResourcePath)): 

131 pathDir = ResourcePath(pathDir, forceDirectory=True) 

132 file = pathDir.join(configFile) 

133 if file.exists(): 

134 self.filesRead.append(file) 

135 self._updateWithOtherConfigFile(file) 

136 break 

137 else: 

138 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

139 else: 

140 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

141 

142 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None: 

143 """Override for base class method. 

144 

145 Parameters 

146 ---------- 

147 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

148 Entity that can be converted to a `ConfigSubset`. 

149 """ 

150 # Use this class to read the defaults so that subsetting can happen 

151 # correctly. 

152 externalConfig = type(self)(file, validate=False) 

153 self.update(externalConfig) 

154 

155 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

156 """Process the 'skypix' section of the configuration. 

157 

158 Yields a construction visitor for each `SkyPixSystem`. 

159 

160 Yields 

161 ------ 

162 visitor : `DimensionConstructionVisitor` 

163 Object that adds a skypix system and its dimensions to an 

164 under-construction `DimensionUniverse`. 

165 """ 

166 config = self["skypix"] 

167 systemNames = set(config.keys()) 

168 systemNames.remove("common") 

169 for systemName in sorted(systemNames): 

170 subconfig = config[systemName] 

171 pixelizationClassName = subconfig["class"] 

172 maxLevel = subconfig.get("max_level", 24) 

173 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

174 

175 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

176 """Process the 'elements' section of the configuration. 

177 

178 Yields a construction visitor for each `StandardDimension` or 

179 `StandardDimensionCombination`. 

180 

181 Yields 

182 ------ 

183 visitor : `DimensionConstructionVisitor` 

184 Object that adds a `StandardDimension` or 

185 `StandardDimensionCombination` to an under-construction 

186 `DimensionUniverse`. 

187 """ 

188 for name, subconfig in self["elements"].items(): 

189 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

190 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

191 if uniqueKeys: 

192 uniqueKeys[0].primaryKey = True 

193 if subconfig.get("governor", False): 

194 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

195 if not unsupported.isdisjoint(subconfig.keys()): 

196 raise RuntimeError( 

197 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

198 ) 

199 if not subconfig.get("cached", True): 

200 raise RuntimeError(f"Governor dimension {name} is always cached.") 

201 yield GovernorDimensionConstructionVisitor( 

202 name=name, 

203 storage=subconfig["storage"], 

204 metadata=metadata, 

205 uniqueKeys=uniqueKeys, 

206 ) 

207 else: 

208 yield DatabaseDimensionElementConstructionVisitor( 

209 name=name, 

210 storage=subconfig["storage"], 

211 required=set(subconfig.get("requires", ())), 

212 implied=set(subconfig.get("implies", ())), 

213 metadata=metadata, 

214 alwaysJoin=subconfig.get("always_join", False), 

215 uniqueKeys=uniqueKeys, 

216 ) 

217 

218 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

219 """Process the 'topology' section of the configuration. 

220 

221 Yields a construction visitor for each `StandardTopologicalFamily`. 

222 

223 Yields 

224 ------ 

225 visitor : `DimensionConstructionVisitor` 

226 Object that adds a `StandardTopologicalFamily` to an 

227 under-construction `DimensionUniverse` and updates its member 

228 `DimensionElement` instances. 

229 """ 

230 for spaceName, subconfig in self.get("topology", {}).items(): 

231 space = TopologicalSpace.__members__[spaceName.upper()] 

232 for name, members in subconfig.items(): 

233 yield DatabaseTopologicalFamilyConstructionVisitor( 

234 name=name, 

235 space=space, 

236 members=members, 

237 ) 

238 

239 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

240 """Process the 'packers' section of the configuration. 

241 

242 Yields construction visitors for each `DimensionPackerFactory`. 

243 

244 Yields 

245 ------ 

246 visitor : `DimensionConstructionVisitor` 

247 Object that adds a `DinmensionPackerFactory` to an 

248 under-construction `DimensionUniverse`. 

249 """ 

250 for name, subconfig in self["packers"].items(): 

251 yield DimensionPackerConstructionVisitor( 

252 name=name, 

253 clsName=subconfig["cls"], 

254 fixed=subconfig["fixed"], 

255 dimensions=subconfig["dimensions"], 

256 ) 

257 

258 def makeBuilder(self) -> DimensionConstructionBuilder: 

259 """Construct a `DinmensionConstructionBuilder`. 

260 

261 The builder will reflect this configuration. 

262 

263 Returns 

264 ------- 

265 builder : `DimensionConstructionBuilder` 

266 A builder object populated with all visitors from this 

267 configuration. The `~DimensionConstructionBuilder.finish` method 

268 will not have been called. 

269 """ 

270 builder = DimensionConstructionBuilder( 

271 self["version"], 

272 self["skypix", "common"], 

273 self, 

274 namespace=self.get("namespace", _DEFAULT_NAMESPACE), 

275 ) 

276 builder.update(self._extractSkyPixVisitors()) 

277 builder.update(self._extractElementVisitors()) 

278 builder.update(self._extractTopologyVisitors()) 

279 builder.update(self._extractPackerVisitors()) 

280 return builder