Coverage for python/lsst/daf/butler/core/dimensions/_packer.py: 54%

68 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("DimensionPacker",) 

31 

32import warnings 

33from abc import ABCMeta, abstractmethod 

34from collections.abc import Iterable, Set 

35from typing import TYPE_CHECKING, Any 

36 

37from deprecated.sphinx import deprecated 

38from lsst.utils import doImportType 

39 

40from ._coordinate import DataCoordinate, DataId 

41from ._graph import DimensionGraph 

42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

43 

44if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

45 from ._universe import DimensionUniverse 

46 

47 

48class DimensionPacker(metaclass=ABCMeta): 

49 """Class for going from `DataCoordinate` to packed integer ID and back. 

50 

51 An abstract base class for bidirectional mappings between a 

52 `DataCoordinate` and a packed integer ID. 

53 

54 Parameters 

55 ---------- 

56 fixed : `DataCoordinate` 

57 Expanded data ID for the dimensions whose values must remain fixed 

58 (to these values) in all calls to `pack`, and are used in the results 

59 of calls to `unpack`. Subclasses are permitted to require that 

60 ``fixed.hasRecords()`` return `True`. 

61 dimensions : `DimensionGraph` 

62 The dimensions of data IDs packed by this instance. 

63 """ 

64 

65 def __init__(self, fixed: DataCoordinate, dimensions: DimensionGraph): 

66 self.fixed = fixed 

67 self.dimensions = dimensions 

68 

69 @property 

70 def universe(self) -> DimensionUniverse: 

71 """Graph containing all known dimensions (`DimensionUniverse`).""" 

72 return self.fixed.universe 

73 

74 @property 

75 @abstractmethod 

76 def maxBits(self) -> int: 

77 """Return The maximum number of nonzero bits in the packed ID. 

78 

79 This packed ID will be returned by 

80 `~DimensionPacker.pack` (`int`). 

81 

82 Must be implemented by all concrete derived classes. May return 

83 `None` to indicate that there is no maximum. 

84 """ 

85 raise NotImplementedError() 

86 

87 @abstractmethod 

88 def _pack(self, dataId: DataCoordinate) -> int: 

89 """Abstract implementation for `~DimensionPacker.pack`. 

90 

91 Must be implemented by all concrete derived classes. 

92 

93 Parameters 

94 ---------- 

95 dataId : `DataCoordinate` 

96 Dictionary-like object identifying (at least) all packed 

97 dimensions associated with this packer. Guaranteed to be a true 

98 `DataCoordinate`, not an informal data ID 

99 

100 Returns 

101 ------- 

102 packed : `int` 

103 Packed integer ID. 

104 """ 

105 raise NotImplementedError() 

106 

107 def pack( 

108 self, dataId: DataId | None = None, *, returnMaxBits: bool = False, **kwargs: Any 

109 ) -> tuple[int, int] | int: 

110 """Pack the given data ID into a single integer. 

111 

112 Parameters 

113 ---------- 

114 dataId : `DataId` 

115 Data ID to pack. Values for any keys also present in the "fixed" 

116 data ID passed at construction must be the same as the values 

117 passed at construction. 

118 returnMaxBits : `bool` 

119 If `True`, return a tuple of ``(packed, self.maxBits)``. 

120 **kwargs 

121 Additional keyword arguments forwarded to 

122 `DataCoordinate.standardize`. 

123 

124 Returns 

125 ------- 

126 packed : `int` 

127 Packed integer ID. 

128 maxBits : `int`, optional 

129 Maximum number of nonzero bits in ``packed``. Not returned unless 

130 ``returnMaxBits`` is `True`. 

131 

132 Notes 

133 ----- 

134 Should not be overridden by derived class 

135 (`~DimensionPacker._pack` should be overridden instead). 

136 """ 

137 dataId = DataCoordinate.standardize( 

138 dataId, **kwargs, universe=self.fixed.universe, defaults=self.fixed 

139 ) 

140 if dataId.subset(self.fixed.graph) != self.fixed: 

141 raise ValueError(f"Data ID packer expected a data ID consistent with {self.fixed}, got {dataId}.") 

142 packed = self._pack(dataId) 

143 if returnMaxBits: 

144 return packed, self.maxBits 

145 else: 

146 return packed 

147 

148 @abstractmethod 

149 def unpack(self, packedId: int) -> DataCoordinate: 

150 """Unpack an ID produced by `pack` into a full `DataCoordinate`. 

151 

152 Must be implemented by all concrete derived classes. 

153 

154 Parameters 

155 ---------- 

156 packedId : `int` 

157 The result of a call to `~DimensionPacker.pack` on either 

158 ``self`` or an identically-constructed packer instance. 

159 

160 Returns 

161 ------- 

162 dataId : `DataCoordinate` 

163 Dictionary-like ID that uniquely identifies all covered 

164 dimensions. 

165 """ 

166 raise NotImplementedError() 

167 

168 # Class attributes below are shadowed by instance attributes, and are 

169 # present just to hold the docstrings for those instance attributes. 

170 

171 fixed: DataCoordinate 

172 """The dimensions provided to the packer at construction 

173 (`DataCoordinate`) 

174 

175 The packed ID values are only unique and reversible with these 

176 dimensions held fixed. 

177 """ 

178 

179 dimensions: DimensionGraph 

180 """The dimensions of data IDs packed by this instance (`DimensionGraph`). 

181 """ 

182 

183 

184# TODO: Remove this class on DM-38687. 

185@deprecated( 

186 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

187 version="v26", 

188 category=FutureWarning, 

189) 

190class DimensionPackerFactory: 

191 """A factory class for `DimensionPacker` instances. 

192 

193 Can be constructed from configuration. 

194 

195 This class is primarily intended for internal use by `DimensionUniverse`. 

196 

197 Parameters 

198 ---------- 

199 clsName : `str` 

200 Fully-qualified name of the packer class this factory constructs. 

201 fixed : `~collections.abc.Set` [ `str` ] 

202 Names of dimensions whose values must be provided to the packer when it 

203 is constructed. This will be expanded lazily into a `DimensionGraph` 

204 prior to `DimensionPacker` construction. 

205 dimensions : `~collections.abc.Set` [ `str` ] 

206 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

207 This will be expanded lazily into a `DimensionGraph` prior to 

208 `DimensionPacker` construction. 

209 """ 

210 

211 def __init__( 

212 self, 

213 clsName: str, 

214 fixed: Set[str], 

215 dimensions: Set[str], 

216 ): 

217 # We defer turning these into DimensionGraph objects until first use 

218 # because __init__ is called before a DimensionUniverse exists, and 

219 # DimensionGraph instances can only be constructed afterwards. 

220 self._fixed: Set[str] | DimensionGraph = fixed 

221 self._dimensions: Set[str] | DimensionGraph = dimensions 

222 self._clsName = clsName 

223 self._cls: type[DimensionPacker] | None = None 

224 

225 def __call__(self, universe: DimensionUniverse, fixed: DataCoordinate) -> DimensionPacker: 

226 """Construct a `DimensionPacker` instance for the given fixed data ID. 

227 

228 Parameters 

229 ---------- 

230 fixed : `DataCoordinate` 

231 Data ID that provides values for the "fixed" dimensions of the 

232 packer. Must be expanded with all metadata known to the 

233 `Registry`. ``fixed.hasRecords()`` must return `True`. 

234 """ 

235 # Construct DimensionGraph instances if necessary on first use. 

236 # See related comment in __init__. 

237 if not isinstance(self._fixed, DimensionGraph): 

238 self._fixed = universe.extract(self._fixed) 

239 if not isinstance(self._dimensions, DimensionGraph): 

240 self._dimensions = universe.extract(self._dimensions) 

241 assert fixed.graph.issuperset(self._fixed) 

242 if self._cls is None: 

243 packer_class = doImportType(self._clsName) 

244 assert not isinstance( 

245 packer_class, DimensionPacker 

246 ), f"Packer class {self._clsName} must be a DimensionPacker." 

247 self._cls = packer_class 

248 return self._cls(fixed, self._dimensions) 

249 

250 

251# TODO: Remove this class on DM-38687. 

252@deprecated( 

253 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

254 version="v26", 

255 category=FutureWarning, 

256) 

257class DimensionPackerConstructionVisitor(DimensionConstructionVisitor): 

258 """Builder visitor for a single `DimensionPacker`. 

259 

260 A single `DimensionPackerConstructionVisitor` should be added to a 

261 `DimensionConstructionBuilder` for each `DimensionPackerFactory` that 

262 should be added to a universe. 

263 

264 Parameters 

265 ---------- 

266 name : `str` 

267 Name used to identify this configuration of the packer in a 

268 `DimensionUniverse`. 

269 clsName : `str` 

270 Fully-qualified name of a `DimensionPacker` subclass. 

271 fixed : `~collections.abc.Iterable` [ `str` ] 

272 Names of dimensions whose values must be provided to the packer when it 

273 is constructed. This will be expanded lazily into a `DimensionGraph` 

274 prior to `DimensionPacker` construction. 

275 dimensions : `~collections.abc.Iterable` [ `str` ] 

276 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

277 This will be expanded lazily into a `DimensionGraph` prior to 

278 `DimensionPacker` construction. 

279 """ 

280 

281 def __init__(self, name: str, clsName: str, fixed: Iterable[str], dimensions: Iterable[str]): 

282 super().__init__(name) 

283 self._fixed = set(fixed) 

284 self._dimensions = set(dimensions) 

285 self._clsName = clsName 

286 

287 def hasDependenciesIn(self, others: Set[str]) -> bool: 

288 # Docstring inherited from DimensionConstructionVisitor. 

289 return False 

290 

291 def visit(self, builder: DimensionConstructionBuilder) -> None: 

292 # Docstring inherited from DimensionConstructionVisitor. 

293 with warnings.catch_warnings(): 

294 # Don't warn when deprecated code calls other deprecated code. 

295 warnings.simplefilter("ignore", FutureWarning) 

296 builder.packers[self.name] = DimensionPackerFactory( 

297 clsName=self._clsName, 

298 fixed=self._fixed, 

299 dimensions=self._dimensions, 

300 )