Coverage for python/lsst/daf/butler/dimensions/_packer.py: 58%

67 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-01 11:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("DimensionPacker",) 

31 

32import warnings 

33from abc import ABCMeta, abstractmethod 

34from collections.abc import Iterable, Set 

35from typing import TYPE_CHECKING, Any 

36 

37from deprecated.sphinx import deprecated 

38from lsst.utils import doImportType 

39 

40from ._coordinate import DataCoordinate, DataId 

41from ._graph import DimensionGraph, DimensionGroup 

42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

43 

44if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

45 from ._universe import DimensionUniverse 

46 

47 

48class DimensionPacker(metaclass=ABCMeta): 

49 """Class for going from `DataCoordinate` to packed integer ID and back. 

50 

51 An abstract base class for bidirectional mappings between a 

52 `DataCoordinate` and a packed integer ID. 

53 

54 Parameters 

55 ---------- 

56 fixed : `DataCoordinate` 

57 Expanded data ID for the dimensions whose values must remain fixed 

58 (to these values) in all calls to `pack`, and are used in the results 

59 of calls to `unpack`. Subclasses may ignore particular dimensions, and 

60 are permitted to require that ``fixed.hasRecords()`` return `True`. 

61 dimensions : `DimensionGroup` or `DimensionGraph` 

62 The dimensions of data IDs packed by this instance. Only 

63 `DimensionGroup` will be supported after v27. 

64 """ 

65 

66 def __init__(self, fixed: DataCoordinate, dimensions: DimensionGroup | DimensionGraph): 

67 self.fixed = fixed 

68 self._dimensions = self.fixed.universe.conform(dimensions) 

69 

70 @property 

71 def universe(self) -> DimensionUniverse: 

72 """Graph containing all known dimensions (`DimensionUniverse`).""" 

73 return self.fixed.universe 

74 

75 @property 

76 def dimensions(self) -> DimensionGraph: 

77 """The dimensions of data IDs packed by this instance 

78 (`DimensionGraph`). 

79 

80 After v27 this will be a `DimensionGroup`. 

81 """ 

82 return self._dimensions._as_graph() 

83 

84 @property 

85 @abstractmethod 

86 def maxBits(self) -> int: 

87 """Return The maximum number of nonzero bits in the packed ID. 

88 

89 This packed ID will be returned by 

90 `~DimensionPacker.pack` (`int`). 

91 

92 Must be implemented by all concrete derived classes. May return 

93 `None` to indicate that there is no maximum. 

94 """ 

95 raise NotImplementedError() 

96 

97 @abstractmethod 

98 def _pack(self, dataId: DataCoordinate) -> int: 

99 """Abstract implementation for `~DimensionPacker.pack`. 

100 

101 Must be implemented by all concrete derived classes. 

102 

103 Parameters 

104 ---------- 

105 dataId : `DataCoordinate` 

106 Dictionary-like object identifying (at least) all packed 

107 dimensions associated with this packer. Guaranteed to be a true 

108 `DataCoordinate`, not an informal data ID 

109 

110 Returns 

111 ------- 

112 packed : `int` 

113 Packed integer ID. 

114 """ 

115 raise NotImplementedError() 

116 

117 def pack( 

118 self, dataId: DataId | None = None, *, returnMaxBits: bool = False, **kwargs: Any 

119 ) -> tuple[int, int] | int: 

120 """Pack the given data ID into a single integer. 

121 

122 Parameters 

123 ---------- 

124 dataId : `DataId` 

125 Data ID to pack. Values for any keys also present in the "fixed" 

126 data ID passed at construction must be the same as the values 

127 passed at construction, but in general you must still specify 

128 those keys. 

129 returnMaxBits : `bool` 

130 If `True`, return a tuple of ``(packed, self.maxBits)``. 

131 **kwargs 

132 Additional keyword arguments are treated like additional key-value 

133 pairs in ``dataId``. 

134 

135 Returns 

136 ------- 

137 packed : `int` 

138 Packed integer ID. 

139 maxBits : `int`, optional 

140 Maximum number of nonzero bits in ``packed``. Not returned unless 

141 ``returnMaxBits`` is `True`. 

142 

143 Notes 

144 ----- 

145 Should not be overridden by derived class 

146 (`~DimensionPacker._pack` should be overridden instead). 

147 """ 

148 dataId = DataCoordinate.standardize( 

149 dataId, **kwargs, universe=self.fixed.universe, defaults=self.fixed 

150 ) 

151 if dataId.subset(self.fixed.dimensions) != self.fixed: 

152 raise ValueError(f"Data ID packer expected a data ID consistent with {self.fixed}, got {dataId}.") 

153 packed = self._pack(dataId) 

154 if returnMaxBits: 

155 return packed, self.maxBits 

156 else: 

157 return packed 

158 

159 @abstractmethod 

160 def unpack(self, packedId: int) -> DataCoordinate: 

161 """Unpack an ID produced by `pack` into a full `DataCoordinate`. 

162 

163 Must be implemented by all concrete derived classes. 

164 

165 Parameters 

166 ---------- 

167 packedId : `int` 

168 The result of a call to `~DimensionPacker.pack` on either 

169 ``self`` or an identically-constructed packer instance. 

170 

171 Returns 

172 ------- 

173 dataId : `DataCoordinate` 

174 Dictionary-like ID that uniquely identifies all covered 

175 dimensions. 

176 """ 

177 raise NotImplementedError() 

178 

179 # Class attributes below are shadowed by instance attributes, and are 

180 # present just to hold the docstrings for those instance attributes. 

181 

182 fixed: DataCoordinate 

183 """The dimensions provided to the packer at construction 

184 (`DataCoordinate`) 

185 

186 The packed ID values are only unique and reversible with these 

187 dimensions held fixed. 

188 """ 

189 

190 

191# TODO: Remove this class on DM-38687. 

192@deprecated( 

193 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

194 version="v26", 

195 category=FutureWarning, 

196) 

197class DimensionPackerFactory: 

198 """A factory class for `DimensionPacker` instances. 

199 

200 Can be constructed from configuration. 

201 

202 This class is primarily intended for internal use by `DimensionUniverse`. 

203 

204 Parameters 

205 ---------- 

206 clsName : `str` 

207 Fully-qualified name of the packer class this factory constructs. 

208 fixed : `~collections.abc.Set` [ `str` ] 

209 Names of dimensions whose values must be provided to the packer when it 

210 is constructed. This will be expanded lazily into a `DimensionGroup` 

211 prior to `DimensionPacker` construction. 

212 dimensions : `~collections.abc.Set` [ `str` ] 

213 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

214 This will be expanded lazily into a `DimensionGroup` prior to 

215 `DimensionPacker` construction. 

216 """ 

217 

218 def __init__( 

219 self, 

220 clsName: str, 

221 fixed: Set[str], 

222 dimensions: Set[str], 

223 ): 

224 # We defer turning these into DimensionGroup objects until first use 

225 # because __init__ is called before a DimensionUniverse exists, and 

226 # DimensionGroup instances can only be constructed afterwards. 

227 self._fixed: Set[str] | DimensionGroup = fixed 

228 self._dimensions: Set[str] | DimensionGroup = dimensions 

229 self._clsName = clsName 

230 self._cls: type[DimensionPacker] | None = None 

231 

232 def __call__(self, universe: DimensionUniverse, fixed: DataCoordinate) -> DimensionPacker: 

233 """Construct a `DimensionPacker` instance for the given fixed data ID. 

234 

235 Parameters 

236 ---------- 

237 fixed : `DataCoordinate` 

238 Data ID that provides values for the "fixed" dimensions of the 

239 packer. Must be expanded with all metadata known to the 

240 `Registry`. ``fixed.hasRecords()`` must return `True`. 

241 """ 

242 # Construct DimensionGroup instances if necessary on first use. 

243 # See related comment in __init__. 

244 self._fixed = universe.conform(self._fixed) 

245 self._dimensions = universe.conform(self._dimensions) 

246 assert fixed.graph.issuperset(self._fixed) 

247 if self._cls is None: 

248 packer_class = doImportType(self._clsName) 

249 assert not isinstance( 

250 packer_class, DimensionPacker 

251 ), f"Packer class {self._clsName} must be a DimensionPacker." 

252 self._cls = packer_class 

253 return self._cls(fixed, self._dimensions) 

254 

255 

256# TODO: Remove this class on DM-38687. 

257@deprecated( 

258 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

259 version="v26", 

260 category=FutureWarning, 

261) 

262class DimensionPackerConstructionVisitor(DimensionConstructionVisitor): 

263 """Builder visitor for a single `DimensionPacker`. 

264 

265 A single `DimensionPackerConstructionVisitor` should be added to a 

266 `DimensionConstructionBuilder` for each `DimensionPackerFactory` that 

267 should be added to a universe. 

268 

269 Parameters 

270 ---------- 

271 name : `str` 

272 Name used to identify this configuration of the packer in a 

273 `DimensionUniverse`. 

274 clsName : `str` 

275 Fully-qualified name of a `DimensionPacker` subclass. 

276 fixed : `~collections.abc.Iterable` [ `str` ] 

277 Names of dimensions whose values must be provided to the packer when it 

278 is constructed. This will be expanded lazily into a `DimensionGroup` 

279 prior to `DimensionPacker` construction. 

280 dimensions : `~collections.abc.Iterable` [ `str` ] 

281 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

282 This will be expanded lazily into a `DimensionGroup` prior to 

283 `DimensionPacker` construction. 

284 """ 

285 

286 def __init__(self, name: str, clsName: str, fixed: Iterable[str], dimensions: Iterable[str]): 

287 super().__init__(name) 

288 self._fixed = set(fixed) 

289 self._dimensions = set(dimensions) 

290 self._clsName = clsName 

291 

292 def hasDependenciesIn(self, others: Set[str]) -> bool: 

293 # Docstring inherited from DimensionConstructionVisitor. 

294 return False 

295 

296 def visit(self, builder: DimensionConstructionBuilder) -> None: 

297 # Docstring inherited from DimensionConstructionVisitor. 

298 with warnings.catch_warnings(): 

299 # Don't warn when deprecated code calls other deprecated code. 

300 warnings.simplefilter("ignore", FutureWarning) 

301 builder.packers[self.name] = DimensionPackerFactory( 

302 clsName=self._clsName, 

303 fixed=self._fixed, 

304 dimensions=self._dimensions, 

305 )