Coverage for python/lsst/daf/butler/dimensions/_config.py: 24%

74 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("DimensionConfig",) 

31 

32from collections.abc import Iterator, Mapping, Sequence 

33from typing import Any 

34 

35import pydantic 

36from lsst.resources import ResourcePath, ResourcePathExpression 

37 

38from .._config import Config, ConfigSubset 

39from .._topology import TopologicalSpace 

40from ._database import ( 

41 DatabaseDimensionElementConstructionVisitor, 

42 DatabaseTopologicalFamilyConstructionVisitor, 

43) 

44from ._elements import KeyColumnSpec, MetadataColumnSpec 

45from ._governor import GovernorDimensionConstructionVisitor 

46from ._skypix import SkyPixConstructionVisitor 

47from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

48 

49# The default namespace to use on older dimension config files that only 

50# have a version. 

51_DEFAULT_NAMESPACE = "daf_butler" 

52 

53 

54class DimensionConfig(ConfigSubset): 

55 """Configuration that defines a `DimensionUniverse`. 

56 

57 The configuration tree for dimensions is a (nested) dictionary 

58 with five top-level entries: 

59 

60 - version: an integer version number, used as keys in a singleton registry 

61 of all `DimensionUniverse` instances; 

62 

63 - namespace: a string to be associated with the version in the singleton 

64 registry of all `DimensionUnivers` instances; 

65 

66 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

67 along with a special "common" key whose value is the name of a skypix 

68 dimension that is used to relate all other spatial dimensions in the 

69 `Registry` database; 

70 

71 - elements: a nested dictionary whose entries each define 

72 `StandardDimension` or `StandardDimensionCombination`. 

73 

74 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

75 with dictionary values that each define a `StandardTopologicalFamily`. 

76 

77 - packers: ignored. 

78 

79 See the documentation for the linked classes above for more information 

80 on the configuration syntax. 

81 

82 Parameters 

83 ---------- 

84 other : `Config` or `str` or `dict`, optional 

85 Argument specifying the configuration information as understood 

86 by `Config`. If `None` is passed then defaults are loaded from 

87 "dimensions.yaml", otherwise defaults are not loaded. 

88 validate : `bool`, optional 

89 If `True` required keys will be checked to ensure configuration 

90 consistency. 

91 searchPaths : `list` or `tuple`, optional 

92 Explicit additional paths to search for defaults. They should 

93 be supplied in priority order. These paths have higher priority 

94 than those read from the environment in 

95 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

96 the local file system or URIs, `lsst.resources.ResourcePath`. 

97 """ 

98 

99 requiredKeys = ("version", "elements", "skypix") 

100 defaultConfigFile = "dimensions.yaml" 

101 

102 def __init__( 

103 self, 

104 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None, 

105 validate: bool = True, 

106 searchPaths: Sequence[ResourcePathExpression] | None = None, 

107 ): 

108 # if argument is not None then do not load/merge defaults 

109 mergeDefaults = other is None 

110 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

111 

112 def _updateWithConfigsFromPath( 

113 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str 

114 ) -> None: 

115 """Search the supplied paths reading config from first found. 

116 

117 Raises 

118 ------ 

119 FileNotFoundError 

120 Raised if config file is not found in any of given locations. 

121 

122 Notes 

123 ----- 

124 This method overrides base class method with different behavior. 

125 Instead of merging all found files into a single configuration it 

126 finds first matching file and reads it. 

127 """ 

128 uri = ResourcePath(configFile) 

129 if uri.isabs() and uri.exists(): 

130 # Assume this resource exists 

131 self._updateWithOtherConfigFile(configFile) 

132 self.filesRead.append(configFile) 

133 else: 

134 for pathDir in searchPaths: 

135 if isinstance(pathDir, str | ResourcePath): 

136 pathDir = ResourcePath(pathDir, forceDirectory=True) 

137 file = pathDir.join(configFile) 

138 if file.exists(): 

139 self.filesRead.append(file) 

140 self._updateWithOtherConfigFile(file) 

141 break 

142 else: 

143 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

144 else: 

145 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

146 

147 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None: 

148 """Override for base class method. 

149 

150 Parameters 

151 ---------- 

152 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

153 Entity that can be converted to a `ConfigSubset`. 

154 """ 

155 # Use this class to read the defaults so that subsetting can happen 

156 # correctly. 

157 externalConfig = type(self)(file, validate=False) 

158 self.update(externalConfig) 

159 

160 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

161 """Process the 'skypix' section of the configuration. 

162 

163 Yields a construction visitor for each `SkyPixSystem`. 

164 

165 Yields 

166 ------ 

167 visitor : `DimensionConstructionVisitor` 

168 Object that adds a skypix system and its dimensions to an 

169 under-construction `DimensionUniverse`. 

170 """ 

171 config = self["skypix"] 

172 systemNames = set(config.keys()) 

173 systemNames.remove("common") 

174 for systemName in sorted(systemNames): 

175 subconfig = config[systemName] 

176 pixelizationClassName = subconfig["class"] 

177 maxLevel = subconfig.get("max_level", 24) 

178 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

179 

180 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

181 """Process the 'elements' section of the configuration. 

182 

183 Yields a construction visitor for each `StandardDimension` or 

184 `StandardDimensionCombination`. 

185 

186 Yields 

187 ------ 

188 visitor : `DimensionConstructionVisitor` 

189 Object that adds a `StandardDimension` or 

190 `StandardDimensionCombination` to an under-construction 

191 `DimensionUniverse`. 

192 """ 

193 # MyPy is confused by the typing.Annotated usage and/or how 

194 # Pydantic annotated TypeAdapter. 

195 key_adapter: pydantic.TypeAdapter[KeyColumnSpec] = pydantic.TypeAdapter( # type: ignore 

196 KeyColumnSpec # type: ignore 

197 ) 

198 metadata_adapter: pydantic.TypeAdapter[MetadataColumnSpec] = pydantic.TypeAdapter( # type: ignore 

199 MetadataColumnSpec # type: ignore 

200 ) 

201 for name, subconfig in self["elements"].items(): 

202 metadata_columns = [metadata_adapter.validate_python(c) for c in subconfig.get("metadata", ())] 

203 unique_keys = [key_adapter.validate_python(c) for c in subconfig.get("keys", ())] 

204 for unique_key in unique_keys: 

205 unique_key.nullable = False 

206 if subconfig.get("governor", False): 

207 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

208 if not unsupported.isdisjoint(subconfig.keys()): 

209 raise RuntimeError( 

210 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

211 ) 

212 if not subconfig.get("cached", True): 

213 raise RuntimeError(f"Governor dimension {name} is always cached.") 

214 yield GovernorDimensionConstructionVisitor( 

215 name=name, 

216 storage=subconfig["storage"], 

217 metadata_columns=metadata_columns, 

218 unique_keys=unique_keys, 

219 doc=subconfig.get("doc", ""), 

220 ) 

221 else: 

222 yield DatabaseDimensionElementConstructionVisitor( 

223 name=name, 

224 storage=subconfig["storage"], 

225 required=set(subconfig.get("requires", ())), 

226 implied=set(subconfig.get("implies", ())), 

227 metadata_columns=metadata_columns, 

228 alwaysJoin=subconfig.get("always_join", False), 

229 unique_keys=unique_keys, 

230 populated_by=subconfig.get("populated_by", None), 

231 doc=subconfig.get("doc", ""), 

232 ) 

233 

234 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

235 """Process the 'topology' section of the configuration. 

236 

237 Yields a construction visitor for each `StandardTopologicalFamily`. 

238 

239 Yields 

240 ------ 

241 visitor : `DimensionConstructionVisitor` 

242 Object that adds a `StandardTopologicalFamily` to an 

243 under-construction `DimensionUniverse` and updates its member 

244 `DimensionElement` instances. 

245 """ 

246 for spaceName, subconfig in self.get("topology", {}).items(): 

247 space = TopologicalSpace.__members__[spaceName.upper()] 

248 for name, members in subconfig.items(): 

249 yield DatabaseTopologicalFamilyConstructionVisitor( 

250 name=name, 

251 space=space, 

252 members=members, 

253 ) 

254 

255 def makeBuilder(self) -> DimensionConstructionBuilder: 

256 """Construct a `DinmensionConstructionBuilder`. 

257 

258 The builder will reflect this configuration. 

259 

260 Returns 

261 ------- 

262 builder : `DimensionConstructionBuilder` 

263 A builder object populated with all visitors from this 

264 configuration. The `~DimensionConstructionBuilder.finish` method 

265 will not have been called. 

266 """ 

267 builder = DimensionConstructionBuilder( 

268 self["version"], 

269 self["skypix", "common"], 

270 self, 

271 namespace=self.get("namespace", _DEFAULT_NAMESPACE), 

272 ) 

273 builder.update(self._extractSkyPixVisitors()) 

274 builder.update(self._extractElementVisitors()) 

275 builder.update(self._extractTopologyVisitors()) 

276 return builder