Coverage for python/lsst/daf/butler/dimensions/_config.py: 23%

79 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("DimensionConfig",) 

31 

32import warnings 

33from collections.abc import Iterator, Mapping, Sequence 

34from typing import Any 

35 

36from lsst.resources import ResourcePath, ResourcePathExpression 

37 

38from .. import ddl 

39from .._config import Config, ConfigSubset 

40from .._topology import TopologicalSpace 

41from ._database import ( 

42 DatabaseDimensionElementConstructionVisitor, 

43 DatabaseTopologicalFamilyConstructionVisitor, 

44) 

45from ._governor import GovernorDimensionConstructionVisitor 

46from ._packer import DimensionPackerConstructionVisitor 

47from ._skypix import SkyPixConstructionVisitor 

48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

49 

50# The default namespace to use on older dimension config files that only 

51# have a version. 

52_DEFAULT_NAMESPACE = "daf_butler" 

53 

54 

55class DimensionConfig(ConfigSubset): 

56 """Configuration that defines a `DimensionUniverse`. 

57 

58 The configuration tree for dimensions is a (nested) dictionary 

59 with five top-level entries: 

60 

61 - version: an integer version number, used as keys in a singleton registry 

62 of all `DimensionUniverse` instances; 

63 

64 - namespace: a string to be associated with the version in the singleton 

65 registry of all `DimensionUnivers` instances; 

66 

67 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

68 along with a special "common" key whose value is the name of a skypix 

69 dimension that is used to relate all other spatial dimensions in the 

70 `Registry` database; 

71 

72 - elements: a nested dictionary whose entries each define 

73 `StandardDimension` or `StandardDimensionCombination`. 

74 

75 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

76 with dictionary values that each define a `StandardTopologicalFamily`. 

77 

78 - packers: a nested dictionary whose entries define factories for a 

79 `DimensionPacker` instance. 

80 

81 See the documentation for the linked classes above for more information 

82 on the configuration syntax. 

83 

84 Parameters 

85 ---------- 

86 other : `Config` or `str` or `dict`, optional 

87 Argument specifying the configuration information as understood 

88 by `Config`. If `None` is passed then defaults are loaded from 

89 "dimensions.yaml", otherwise defaults are not loaded. 

90 validate : `bool`, optional 

91 If `True` required keys will be checked to ensure configuration 

92 consistency. 

93 searchPaths : `list` or `tuple`, optional 

94 Explicit additional paths to search for defaults. They should 

95 be supplied in priority order. These paths have higher priority 

96 than those read from the environment in 

97 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

98 the local file system or URIs, `lsst.resources.ResourcePath`. 

99 """ 

100 

101 requiredKeys = ("version", "elements", "skypix") 

102 defaultConfigFile = "dimensions.yaml" 

103 

104 def __init__( 

105 self, 

106 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None, 

107 validate: bool = True, 

108 searchPaths: Sequence[ResourcePathExpression] | None = None, 

109 ): 

110 # if argument is not None then do not load/merge defaults 

111 mergeDefaults = other is None 

112 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

113 

114 def _updateWithConfigsFromPath( 

115 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str 

116 ) -> None: 

117 """Search the supplied paths reading config from first found. 

118 

119 Raises 

120 ------ 

121 FileNotFoundError 

122 Raised if config file is not found in any of given locations. 

123 

124 Notes 

125 ----- 

126 This method overrides base class method with different behavior. 

127 Instead of merging all found files into a single configuration it 

128 finds first matching file and reads it. 

129 """ 

130 uri = ResourcePath(configFile) 

131 if uri.isabs() and uri.exists(): 

132 # Assume this resource exists 

133 self._updateWithOtherConfigFile(configFile) 

134 self.filesRead.append(configFile) 

135 else: 

136 for pathDir in searchPaths: 

137 if isinstance(pathDir, str | ResourcePath): 

138 pathDir = ResourcePath(pathDir, forceDirectory=True) 

139 file = pathDir.join(configFile) 

140 if file.exists(): 

141 self.filesRead.append(file) 

142 self._updateWithOtherConfigFile(file) 

143 break 

144 else: 

145 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

146 else: 

147 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

148 

149 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None: 

150 """Override for base class method. 

151 

152 Parameters 

153 ---------- 

154 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

155 Entity that can be converted to a `ConfigSubset`. 

156 """ 

157 # Use this class to read the defaults so that subsetting can happen 

158 # correctly. 

159 externalConfig = type(self)(file, validate=False) 

160 self.update(externalConfig) 

161 

162 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

163 """Process the 'skypix' section of the configuration. 

164 

165 Yields a construction visitor for each `SkyPixSystem`. 

166 

167 Yields 

168 ------ 

169 visitor : `DimensionConstructionVisitor` 

170 Object that adds a skypix system and its dimensions to an 

171 under-construction `DimensionUniverse`. 

172 """ 

173 config = self["skypix"] 

174 systemNames = set(config.keys()) 

175 systemNames.remove("common") 

176 for systemName in sorted(systemNames): 

177 subconfig = config[systemName] 

178 pixelizationClassName = subconfig["class"] 

179 maxLevel = subconfig.get("max_level", 24) 

180 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

181 

182 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

183 """Process the 'elements' section of the configuration. 

184 

185 Yields a construction visitor for each `StandardDimension` or 

186 `StandardDimensionCombination`. 

187 

188 Yields 

189 ------ 

190 visitor : `DimensionConstructionVisitor` 

191 Object that adds a `StandardDimension` or 

192 `StandardDimensionCombination` to an under-construction 

193 `DimensionUniverse`. 

194 """ 

195 for name, subconfig in self["elements"].items(): 

196 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

197 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

198 if uniqueKeys: 

199 uniqueKeys[0].primaryKey = True 

200 if subconfig.get("governor", False): 

201 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

202 if not unsupported.isdisjoint(subconfig.keys()): 

203 raise RuntimeError( 

204 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

205 ) 

206 if not subconfig.get("cached", True): 

207 raise RuntimeError(f"Governor dimension {name} is always cached.") 

208 yield GovernorDimensionConstructionVisitor( 

209 name=name, 

210 storage=subconfig["storage"], 

211 metadata=metadata, 

212 uniqueKeys=uniqueKeys, 

213 ) 

214 else: 

215 yield DatabaseDimensionElementConstructionVisitor( 

216 name=name, 

217 storage=subconfig["storage"], 

218 required=set(subconfig.get("requires", ())), 

219 implied=set(subconfig.get("implies", ())), 

220 metadata=metadata, 

221 alwaysJoin=subconfig.get("always_join", False), 

222 uniqueKeys=uniqueKeys, 

223 populated_by=subconfig.get("populated_by", None), 

224 ) 

225 

226 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

227 """Process the 'topology' section of the configuration. 

228 

229 Yields a construction visitor for each `StandardTopologicalFamily`. 

230 

231 Yields 

232 ------ 

233 visitor : `DimensionConstructionVisitor` 

234 Object that adds a `StandardTopologicalFamily` to an 

235 under-construction `DimensionUniverse` and updates its member 

236 `DimensionElement` instances. 

237 """ 

238 for spaceName, subconfig in self.get("topology", {}).items(): 

239 space = TopologicalSpace.__members__[spaceName.upper()] 

240 for name, members in subconfig.items(): 

241 yield DatabaseTopologicalFamilyConstructionVisitor( 

242 name=name, 

243 space=space, 

244 members=members, 

245 ) 

246 

247 # TODO: remove this method and callers on DM-38687. 

248 # Note that the corresponding entries in the dimensions config should 

249 # not be removed at that time, because that's formally a schema migration. 

250 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

251 """Process the 'packers' section of the configuration. 

252 

253 Yields construction visitors for each `DimensionPackerFactory`. 

254 

255 Yields 

256 ------ 

257 visitor : `DimensionConstructionVisitor` 

258 Object that adds a `DinmensionPackerFactory` to an 

259 under-construction `DimensionUniverse`. 

260 """ 

261 with warnings.catch_warnings(): 

262 # Don't warn when deprecated code calls other deprecated code. 

263 warnings.simplefilter("ignore", FutureWarning) 

264 for name, subconfig in self["packers"].items(): 

265 yield DimensionPackerConstructionVisitor( 

266 name=name, 

267 clsName=subconfig["cls"], 

268 fixed=subconfig["fixed"], 

269 dimensions=subconfig["dimensions"], 

270 ) 

271 

272 def makeBuilder(self) -> DimensionConstructionBuilder: 

273 """Construct a `DinmensionConstructionBuilder`. 

274 

275 The builder will reflect this configuration. 

276 

277 Returns 

278 ------- 

279 builder : `DimensionConstructionBuilder` 

280 A builder object populated with all visitors from this 

281 configuration. The `~DimensionConstructionBuilder.finish` method 

282 will not have been called. 

283 """ 

284 builder = DimensionConstructionBuilder( 

285 self["version"], 

286 self["skypix", "common"], 

287 self, 

288 namespace=self.get("namespace", _DEFAULT_NAMESPACE), 

289 ) 

290 builder.update(self._extractSkyPixVisitors()) 

291 builder.update(self._extractElementVisitors()) 

292 builder.update(self._extractTopologyVisitors()) 

293 builder.update(self._extractPackerVisitors()) 

294 return builder