Coverage for python/lsst/daf/butler/core/dimensions/_packer.py: 47%

67 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-17 02:31 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("DimensionPacker",) 

25 

26import warnings 

27from abc import ABCMeta, abstractmethod 

28from typing import TYPE_CHECKING, AbstractSet, Any, Iterable, Optional, Tuple, Type, Union 

29 

30from deprecated.sphinx import deprecated 

31from lsst.utils import doImportType 

32 

33from ._coordinate import DataCoordinate, DataId 

34from ._graph import DimensionGraph 

35from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

36 

37if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

38 from ._universe import DimensionUniverse 

39 

40 

41class DimensionPacker(metaclass=ABCMeta): 

42 """Class for going from `DataCoordinate` to packed integer ID and back. 

43 

44 An abstract base class for bidirectional mappings between a 

45 `DataCoordinate` and a packed integer ID. 

46 

47 Parameters 

48 ---------- 

49 fixed : `DataCoordinate` 

50 Expanded data ID for the dimensions whose values must remain fixed 

51 (to these values) in all calls to `pack`, and are used in the results 

52 of calls to `unpack`. Subclasses are permitted to require that 

53 ``fixed.hasRecords()`` return `True`. 

54 dimensions : `DimensionGraph` 

55 The dimensions of data IDs packed by this instance. 

56 """ 

57 

58 def __init__(self, fixed: DataCoordinate, dimensions: DimensionGraph): 

59 self.fixed = fixed 

60 self.dimensions = dimensions 

61 

62 @property 

63 def universe(self) -> DimensionUniverse: 

64 """Graph containing all known dimensions (`DimensionUniverse`).""" 

65 return self.fixed.universe 

66 

67 @property 

68 @abstractmethod 

69 def maxBits(self) -> int: 

70 """Return The maximum number of nonzero bits in the packed ID. 

71 

72 This packed ID will be returned by 

73 `~DimensionPacker.pack` (`int`). 

74 

75 Must be implemented by all concrete derived classes. May return 

76 `None` to indicate that there is no maximum. 

77 """ 

78 raise NotImplementedError() 

79 

80 @abstractmethod 

81 def _pack(self, dataId: DataCoordinate) -> int: 

82 """Abstract implementation for `~DimensionPacker.pack`. 

83 

84 Must be implemented by all concrete derived classes. 

85 

86 Parameters 

87 ---------- 

88 dataId : `DataCoordinate` 

89 Dictionary-like object identifying (at least) all packed 

90 dimensions associated with this packer. Guaranteed to be a true 

91 `DataCoordinate`, not an informal data ID 

92 

93 Returns 

94 ------- 

95 packed : `int` 

96 Packed integer ID. 

97 """ 

98 raise NotImplementedError() 

99 

100 def pack( 

101 self, dataId: DataId | None = None, *, returnMaxBits: bool = False, **kwargs: Any 

102 ) -> Union[Tuple[int, int], int]: 

103 """Pack the given data ID into a single integer. 

104 

105 Parameters 

106 ---------- 

107 dataId : `DataId` 

108 Data ID to pack. Values for any keys also present in the "fixed" 

109 data ID passed at construction must be the same as the values 

110 passed at construction. 

111 returnMaxBits : `bool` 

112 If `True`, return a tuple of ``(packed, self.maxBits)``. 

113 **kwargs 

114 Additional keyword arguments forwarded to 

115 `DataCoordinate.standardize`. 

116 

117 Returns 

118 ------- 

119 packed : `int` 

120 Packed integer ID. 

121 maxBits : `int`, optional 

122 Maximum number of nonzero bits in ``packed``. Not returned unless 

123 ``returnMaxBits`` is `True`. 

124 

125 Notes 

126 ----- 

127 Should not be overridden by derived class 

128 (`~DimensionPacker._pack` should be overridden instead). 

129 """ 

130 dataId = DataCoordinate.standardize( 

131 dataId, **kwargs, universe=self.fixed.universe, defaults=self.fixed 

132 ) 

133 if dataId.subset(self.fixed.graph) != self.fixed: 

134 raise ValueError(f"Data ID packer expected a data ID consistent with {self.fixed}, got {dataId}.") 

135 packed = self._pack(dataId) 

136 if returnMaxBits: 

137 return packed, self.maxBits 

138 else: 

139 return packed 

140 

141 @abstractmethod 

142 def unpack(self, packedId: int) -> DataCoordinate: 

143 """Unpack an ID produced by `pack` into a full `DataCoordinate`. 

144 

145 Must be implemented by all concrete derived classes. 

146 

147 Parameters 

148 ---------- 

149 packedId : `int` 

150 The result of a call to `~DimensionPacker.pack` on either 

151 ``self`` or an identically-constructed packer instance. 

152 

153 Returns 

154 ------- 

155 dataId : `DataCoordinate` 

156 Dictionary-like ID that uniquely identifies all covered 

157 dimensions. 

158 """ 

159 raise NotImplementedError() 

160 

161 # Class attributes below are shadowed by instance attributes, and are 

162 # present just to hold the docstrings for those instance attributes. 

163 

164 fixed: DataCoordinate 

165 """The dimensions provided to the packer at construction 

166 (`DataCoordinate`) 

167 

168 The packed ID values are only unique and reversible with these 

169 dimensions held fixed. 

170 """ 

171 

172 dimensions: DimensionGraph 

173 """The dimensions of data IDs packed by this instance (`DimensionGraph`). 

174 """ 

175 

176 

177# TODO: Remove this class on DM-38687. 

178@deprecated( 

179 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

180 version="v26", 

181 category=FutureWarning, 

182) 

183class DimensionPackerFactory: 

184 """A factory class for `DimensionPacker` instances. 

185 

186 Can be constructed from configuration. 

187 

188 This class is primarily intended for internal use by `DimensionUniverse`. 

189 

190 Parameters 

191 ---------- 

192 clsName : `str` 

193 Fully-qualified name of the packer class this factory constructs. 

194 fixed : `AbstractSet` [ `str` ] 

195 Names of dimensions whose values must be provided to the packer when it 

196 is constructed. This will be expanded lazily into a `DimensionGraph` 

197 prior to `DimensionPacker` construction. 

198 dimensions : `AbstractSet` [ `str` ] 

199 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

200 This will be expanded lazily into a `DimensionGraph` prior to 

201 `DimensionPacker` construction. 

202 """ 

203 

204 def __init__( 

205 self, 

206 clsName: str, 

207 fixed: AbstractSet[str], 

208 dimensions: AbstractSet[str], 

209 ): 

210 # We defer turning these into DimensionGraph objects until first use 

211 # because __init__ is called before a DimensionUniverse exists, and 

212 # DimensionGraph instances can only be constructed afterwards. 

213 self._fixed: Union[AbstractSet[str], DimensionGraph] = fixed 

214 self._dimensions: Union[AbstractSet[str], DimensionGraph] = dimensions 

215 self._clsName = clsName 

216 self._cls: Optional[Type[DimensionPacker]] = None 

217 

218 def __call__(self, universe: DimensionUniverse, fixed: DataCoordinate) -> DimensionPacker: 

219 """Construct a `DimensionPacker` instance for the given fixed data ID. 

220 

221 Parameters 

222 ---------- 

223 fixed : `DataCoordinate` 

224 Data ID that provides values for the "fixed" dimensions of the 

225 packer. Must be expanded with all metadata known to the 

226 `Registry`. ``fixed.hasRecords()`` must return `True`. 

227 """ 

228 # Construct DimensionGraph instances if necessary on first use. 

229 # See related comment in __init__. 

230 if not isinstance(self._fixed, DimensionGraph): 

231 self._fixed = universe.extract(self._fixed) 

232 if not isinstance(self._dimensions, DimensionGraph): 

233 self._dimensions = universe.extract(self._dimensions) 

234 assert fixed.graph.issuperset(self._fixed) 

235 if self._cls is None: 

236 packer_class = doImportType(self._clsName) 

237 assert not isinstance( 

238 packer_class, DimensionPacker 

239 ), f"Packer class {self._clsName} must be a DimensionPacker." 

240 self._cls = packer_class 

241 return self._cls(fixed, self._dimensions) 

242 

243 

244# TODO: Remove this class on DM-38687. 

245@deprecated( 

246 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

247 version="v26", 

248 category=FutureWarning, 

249) 

250class DimensionPackerConstructionVisitor(DimensionConstructionVisitor): 

251 """Builder visitor for a single `DimensionPacker`. 

252 

253 A single `DimensionPackerConstructionVisitor` should be added to a 

254 `DimensionConstructionBuilder` for each `DimensionPackerFactory` that 

255 should be added to a universe. 

256 

257 Parameters 

258 ---------- 

259 name : `str` 

260 Name used to identify this configuration of the packer in a 

261 `DimensionUniverse`. 

262 clsName : `str` 

263 Fully-qualified name of a `DimensionPacker` subclass. 

264 fixed : `Iterable` [ `str` ] 

265 Names of dimensions whose values must be provided to the packer when it 

266 is constructed. This will be expanded lazily into a `DimensionGraph` 

267 prior to `DimensionPacker` construction. 

268 dimensions : `Iterable` [ `str` ] 

269 Names of dimensions whose values are passed to `DimensionPacker.pack`. 

270 This will be expanded lazily into a `DimensionGraph` prior to 

271 `DimensionPacker` construction. 

272 """ 

273 

274 def __init__(self, name: str, clsName: str, fixed: Iterable[str], dimensions: Iterable[str]): 

275 super().__init__(name) 

276 self._fixed = set(fixed) 

277 self._dimensions = set(dimensions) 

278 self._clsName = clsName 

279 

280 def hasDependenciesIn(self, others: AbstractSet[str]) -> bool: 

281 # Docstring inherited from DimensionConstructionVisitor. 

282 return False 

283 

284 def visit(self, builder: DimensionConstructionBuilder) -> None: 

285 # Docstring inherited from DimensionConstructionVisitor. 

286 with warnings.catch_warnings(): 

287 # Don't warn when deprecated code calls other deprecated code. 

288 warnings.simplefilter("ignore", FutureWarning) 

289 builder.packers[self.name] = DimensionPackerFactory( 

290 clsName=self._clsName, 

291 fixed=self._fixed, 

292 dimensions=self._dimensions, 

293 )