Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 23%

79 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 15:14 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionConfig",) 

25 

26import warnings 

27from collections.abc import Iterator, Mapping, Sequence 

28from typing import Any 

29 

30from lsst.resources import ResourcePath, ResourcePathExpression 

31 

32from .. import ddl 

33from .._topology import TopologicalSpace 

34from ..config import Config, ConfigSubset 

35from ._database import ( 

36 DatabaseDimensionElementConstructionVisitor, 

37 DatabaseTopologicalFamilyConstructionVisitor, 

38) 

39from ._governor import GovernorDimensionConstructionVisitor 

40from ._packer import DimensionPackerConstructionVisitor 

41from ._skypix import SkyPixConstructionVisitor 

42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

43 

44# The default namespace to use on older dimension config files that only 

45# have a version. 

46_DEFAULT_NAMESPACE = "daf_butler" 

47 

48 

49class DimensionConfig(ConfigSubset): 

50 """Configuration that defines a `DimensionUniverse`. 

51 

52 The configuration tree for dimensions is a (nested) dictionary 

53 with five top-level entries: 

54 

55 - version: an integer version number, used as keys in a singleton registry 

56 of all `DimensionUniverse` instances; 

57 

58 - namespace: a string to be associated with the version in the singleton 

59 registry of all `DimensionUnivers` instances; 

60 

61 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

62 along with a special "common" key whose value is the name of a skypix 

63 dimension that is used to relate all other spatial dimensions in the 

64 `Registry` database; 

65 

66 - elements: a nested dictionary whose entries each define 

67 `StandardDimension` or `StandardDimensionCombination`. 

68 

69 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

70 with dictionary values that each define a `StandardTopologicalFamily`. 

71 

72 - packers: a nested dictionary whose entries define factories for a 

73 `DimensionPacker` instance. 

74 

75 See the documentation for the linked classes above for more information 

76 on the configuration syntax. 

77 

78 Parameters 

79 ---------- 

80 other : `Config` or `str` or `dict`, optional 

81 Argument specifying the configuration information as understood 

82 by `Config`. If `None` is passed then defaults are loaded from 

83 "dimensions.yaml", otherwise defaults are not loaded. 

84 validate : `bool`, optional 

85 If `True` required keys will be checked to ensure configuration 

86 consistency. 

87 searchPaths : `list` or `tuple`, optional 

88 Explicit additional paths to search for defaults. They should 

89 be supplied in priority order. These paths have higher priority 

90 than those read from the environment in 

91 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

92 the local file system or URIs, `lsst.resources.ResourcePath`. 

93 """ 

94 

95 requiredKeys = ("version", "elements", "skypix") 

96 defaultConfigFile = "dimensions.yaml" 

97 

98 def __init__( 

99 self, 

100 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None, 

101 validate: bool = True, 

102 searchPaths: Sequence[ResourcePathExpression] | None = None, 

103 ): 

104 # if argument is not None then do not load/merge defaults 

105 mergeDefaults = other is None 

106 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

107 

108 def _updateWithConfigsFromPath( 

109 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str 

110 ) -> None: 

111 """Search the supplied paths reading config from first found. 

112 

113 Raises 

114 ------ 

115 FileNotFoundError 

116 Raised if config file is not found in any of given locations. 

117 

118 Notes 

119 ----- 

120 This method overrides base class method with different behavior. 

121 Instead of merging all found files into a single configuration it 

122 finds first matching file and reads it. 

123 """ 

124 uri = ResourcePath(configFile) 

125 if uri.isabs() and uri.exists(): 

126 # Assume this resource exists 

127 self._updateWithOtherConfigFile(configFile) 

128 self.filesRead.append(configFile) 

129 else: 

130 for pathDir in searchPaths: 

131 if isinstance(pathDir, str | ResourcePath): 

132 pathDir = ResourcePath(pathDir, forceDirectory=True) 

133 file = pathDir.join(configFile) 

134 if file.exists(): 

135 self.filesRead.append(file) 

136 self._updateWithOtherConfigFile(file) 

137 break 

138 else: 

139 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

140 else: 

141 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

142 

143 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None: 

144 """Override for base class method. 

145 

146 Parameters 

147 ---------- 

148 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

149 Entity that can be converted to a `ConfigSubset`. 

150 """ 

151 # Use this class to read the defaults so that subsetting can happen 

152 # correctly. 

153 externalConfig = type(self)(file, validate=False) 

154 self.update(externalConfig) 

155 

156 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

157 """Process the 'skypix' section of the configuration. 

158 

159 Yields a construction visitor for each `SkyPixSystem`. 

160 

161 Yields 

162 ------ 

163 visitor : `DimensionConstructionVisitor` 

164 Object that adds a skypix system and its dimensions to an 

165 under-construction `DimensionUniverse`. 

166 """ 

167 config = self["skypix"] 

168 systemNames = set(config.keys()) 

169 systemNames.remove("common") 

170 for systemName in sorted(systemNames): 

171 subconfig = config[systemName] 

172 pixelizationClassName = subconfig["class"] 

173 maxLevel = subconfig.get("max_level", 24) 

174 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel) 

175 

176 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

177 """Process the 'elements' section of the configuration. 

178 

179 Yields a construction visitor for each `StandardDimension` or 

180 `StandardDimensionCombination`. 

181 

182 Yields 

183 ------ 

184 visitor : `DimensionConstructionVisitor` 

185 Object that adds a `StandardDimension` or 

186 `StandardDimensionCombination` to an under-construction 

187 `DimensionUniverse`. 

188 """ 

189 for name, subconfig in self["elements"].items(): 

190 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())] 

191 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())] 

192 if uniqueKeys: 

193 uniqueKeys[0].primaryKey = True 

194 if subconfig.get("governor", False): 

195 unsupported = {"required", "implied", "viewOf", "alwaysJoin"} 

196 if not unsupported.isdisjoint(subconfig.keys()): 

197 raise RuntimeError( 

198 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}." 

199 ) 

200 if not subconfig.get("cached", True): 

201 raise RuntimeError(f"Governor dimension {name} is always cached.") 

202 yield GovernorDimensionConstructionVisitor( 

203 name=name, 

204 storage=subconfig["storage"], 

205 metadata=metadata, 

206 uniqueKeys=uniqueKeys, 

207 ) 

208 else: 

209 yield DatabaseDimensionElementConstructionVisitor( 

210 name=name, 

211 storage=subconfig["storage"], 

212 required=set(subconfig.get("requires", ())), 

213 implied=set(subconfig.get("implies", ())), 

214 metadata=metadata, 

215 alwaysJoin=subconfig.get("always_join", False), 

216 uniqueKeys=uniqueKeys, 

217 populated_by=subconfig.get("populated_by", None), 

218 ) 

219 

220 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

221 """Process the 'topology' section of the configuration. 

222 

223 Yields a construction visitor for each `StandardTopologicalFamily`. 

224 

225 Yields 

226 ------ 

227 visitor : `DimensionConstructionVisitor` 

228 Object that adds a `StandardTopologicalFamily` to an 

229 under-construction `DimensionUniverse` and updates its member 

230 `DimensionElement` instances. 

231 """ 

232 for spaceName, subconfig in self.get("topology", {}).items(): 

233 space = TopologicalSpace.__members__[spaceName.upper()] 

234 for name, members in subconfig.items(): 

235 yield DatabaseTopologicalFamilyConstructionVisitor( 

236 name=name, 

237 space=space, 

238 members=members, 

239 ) 

240 

241 # TODO: remove this method and callers on DM-38687. 

242 # Note that the corresponding entries in the dimensions config should 

243 # not be removed at that time, because that's formally a schema migration. 

244 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]: 

245 """Process the 'packers' section of the configuration. 

246 

247 Yields construction visitors for each `DimensionPackerFactory`. 

248 

249 Yields 

250 ------ 

251 visitor : `DimensionConstructionVisitor` 

252 Object that adds a `DinmensionPackerFactory` to an 

253 under-construction `DimensionUniverse`. 

254 """ 

255 with warnings.catch_warnings(): 

256 # Don't warn when deprecated code calls other deprecated code. 

257 warnings.simplefilter("ignore", FutureWarning) 

258 for name, subconfig in self["packers"].items(): 

259 yield DimensionPackerConstructionVisitor( 

260 name=name, 

261 clsName=subconfig["cls"], 

262 fixed=subconfig["fixed"], 

263 dimensions=subconfig["dimensions"], 

264 ) 

265 

266 def makeBuilder(self) -> DimensionConstructionBuilder: 

267 """Construct a `DinmensionConstructionBuilder`. 

268 

269 The builder will reflect this configuration. 

270 

271 Returns 

272 ------- 

273 builder : `DimensionConstructionBuilder` 

274 A builder object populated with all visitors from this 

275 configuration. The `~DimensionConstructionBuilder.finish` method 

276 will not have been called. 

277 """ 

278 builder = DimensionConstructionBuilder( 

279 self["version"], 

280 self["skypix", "common"], 

281 self, 

282 namespace=self.get("namespace", _DEFAULT_NAMESPACE), 

283 ) 

284 builder.update(self._extractSkyPixVisitors()) 

285 builder.update(self._extractElementVisitors()) 

286 builder.update(self._extractTopologyVisitors()) 

287 builder.update(self._extractPackerVisitors()) 

288 return builder