Coverage for python/lsst/daf/butler/registry/summaries.py: 30%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

101 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "CollectionSummary", 

25 "GovernorDimensionRestriction", 

26) 

27 

28from dataclasses import dataclass 

29import itertools 

30from typing import ( 

31 AbstractSet, 

32 Any, 

33 ItemsView, 

34 Iterable, 

35 Iterator, 

36 List, 

37 Mapping, 

38 Optional, 

39 Set, 

40 Union, 

41 ValuesView, 

42) 

43 

44from ..core import ( 

45 DataCoordinate, 

46 DatasetType, 

47 DimensionUniverse, 

48 GovernorDimension, 

49 NamedKeyDict, 

50 NamedKeyMapping, 

51 NamedValueAbstractSet, 

52 NamedValueSet, 

53) 

54from ..core.utils import iterable 

55 

56 

57class GovernorDimensionRestriction(NamedKeyMapping[GovernorDimension, AbstractSet[str]]): 

58 """A custom mapping that represents a restriction on the values one or 

59 more governor dimensions may take in some context. 

60 

61 Parameters 

62 ---------- 

63 mapping : `NamedKeyDict` [ `GovernorDimension`, `Set` [ `str` ]] 

64 Mapping from governor dimension to the values it may take. Dimensions 

65 not present in the mapping are not constrained at all. 

66 """ 

67 def __init__(self, mapping: NamedKeyDict[GovernorDimension, Set[str]]): 

68 self._mapping = mapping 

69 

70 @classmethod 

71 def makeEmpty(cls, universe: DimensionUniverse) -> GovernorDimensionRestriction: 

72 """Construct a `GovernorDimensionRestriction` that allows no values 

73 for any governor dimension in the given `DimensionUniverse`. 

74 

75 Parameters 

76 ---------- 

77 universe : `DimensionUniverse` 

78 Object that manages all dimensions. 

79 

80 Returns 

81 ------- 

82 restriction : `GovernorDimensionRestriction` 

83 Restriction instance that maps all governor dimensions to an empty 

84 set. 

85 """ 

86 return cls(NamedKeyDict((k, set()) for k in universe.getGovernorDimensions())) 

87 

88 @classmethod 

89 def makeFull(cls) -> GovernorDimensionRestriction: 

90 """Construct a `GovernorDimensionRestriction` that allows any value 

91 for any governor dimension. 

92 

93 Returns 

94 ------- 

95 restriction : `GovernorDimensionRestriction` 

96 Restriction instance that contains no keys, and hence contains 

97 allows any value for any governor dimension. 

98 """ 

99 return cls(NamedKeyDict()) 

100 

101 def __eq__(self, other: Any) -> bool: 

102 if not isinstance(other, GovernorDimensionRestriction): 

103 return False 

104 return self._mapping == other._mapping 

105 

106 def __str__(self) -> str: 

107 return "({})".format( 

108 ", ".join(f"{dimension.name}: {values}" for dimension, values in self._mapping.items()) 

109 ) 

110 

111 def __repr__(self) -> str: 

112 return "GovernorDimensionRestriction({})".format( 

113 ", ".join(f"{dimension.name}={values}" for dimension, values in self._mapping.items()) 

114 ) 

115 

116 def __iter__(self) -> Iterator[GovernorDimension]: 

117 return iter(self._mapping) 

118 

119 def __len__(self) -> int: 

120 return len(self._mapping) 

121 

122 @property 

123 def names(self) -> AbstractSet[str]: 

124 # Docstring inherited. 

125 return self._mapping.names 

126 

127 def keys(self) -> NamedValueAbstractSet[GovernorDimension]: 

128 return self._mapping.keys() 

129 

130 def values(self) -> ValuesView[AbstractSet[str]]: 

131 return self._mapping.values() 

132 

133 def items(self) -> ItemsView[GovernorDimension, AbstractSet[str]]: 

134 return self._mapping.items() 

135 

136 def __getitem__(self, key: Union[str, GovernorDimension]) -> AbstractSet[str]: 

137 return self._mapping[key] 

138 

139 def copy(self) -> GovernorDimensionRestriction: 

140 """Return a deep copy of this object. 

141 

142 Returns 

143 ------- 

144 copy : `GovernorDimensionRestriction` 

145 A copy of ``self`` that can be modified without modifying ``self`` 

146 at all. 

147 """ 

148 return GovernorDimensionRestriction(NamedKeyDict((k, set(v)) for k, v in self.items())) 

149 

150 def add(self, dimension: GovernorDimension, value: str) -> None: 

151 """Add a single dimension value to the restriction. 

152 

153 Parameters 

154 ---------- 

155 dimension : `GovernorDimension` 

156 Dimension to update. 

157 value : `str` 

158 Value to allow for this dimension. 

159 """ 

160 current = self._mapping.get(dimension) 

161 if current is not None: 

162 current.add(value) 

163 

164 def update(self, other: Mapping[GovernorDimension, Union[str, Iterable[str]]]) -> None: 

165 """Update ``self`` to include all dimension values in either ``self`` 

166 or ``other``. 

167 

168 Parameters 

169 ---------- 

170 other : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ] 

171 Mapping to union into ``self``. This may be another 

172 `GovernorDimensionRestriction` or any other mapping from dimension 

173 to `str` or iterable of `str`. 

174 """ 

175 for dimension in (self.keys() - other.keys()): 

176 self._mapping.pop(dimension, None) 

177 for dimension in (self.keys() & other.keys()): 

178 self._mapping[dimension].update(iterable(other[dimension])) 

179 # Dimensions that are in 'other' but not in 'self' are ignored, because 

180 # 'self' says they are already unconstrained. 

181 

182 def union(self, *others: Mapping[GovernorDimension, Union[str, Iterable[str]]] 

183 ) -> GovernorDimensionRestriction: 

184 """Construct a restriction that permits any values permitted by any of 

185 the input restrictions. 

186 

187 Parameters 

188 ---------- 

189 *others : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ] 

190 Mappings to union into ``self``. These may be other 

191 `GovernorDimensionRestriction` instances or any other kind of 

192 mapping from dimension to `str` or iterable of `str`. 

193 

194 Returns 

195 ------- 

196 unioned : `GovernorDimensionRestriction` 

197 New restriction object that represents the union of ``self`` with 

198 ``others``. 

199 """ 

200 result = self.copy() 

201 for other in others: 

202 result.update(other) 

203 return result 

204 

205 def intersection_update(self, other: Mapping[GovernorDimension, Union[str, Iterable[str]]]) -> None: 

206 """Update ``self`` to include only dimension values in both ``self`` 

207 and ``other``. 

208 

209 Parameters 

210 ---------- 

211 other : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ] 

212 Mapping to intersect into ``self``. This may be another 

213 `GovernorDimensionRestriction` or any other mapping from dimension 

214 to `str` or iterable of `str`. 

215 """ 

216 for dimension, values in other.items(): 

217 new_values = set(iterable(values)) 

218 # Yes, this will often result in a (no-op) self-intersection on the 

219 # inner set, but this is easier to read (and obviously more or less 

220 # efficient) than adding a check to avoid it. 

221 self._mapping.setdefault(dimension, new_values).intersection_update(new_values) 

222 

223 def intersection(self, *others: Mapping[GovernorDimension, Union[str, Iterable[str]]] 

224 ) -> GovernorDimensionRestriction: 

225 """Construct a restriction that permits only values permitted by all of 

226 the input restrictions. 

227 

228 Parameters 

229 ---------- 

230 *others : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ] 

231 Mappings to intersect with ``self``. These may be other 

232 `GovernorDimensionRestriction` instances or any other kind of 

233 mapping from dimension to `str` or iterable of `str`. 

234 Returns 

235 ------- 

236 intersection : `GovernorDimensionRestriction` 

237 New restriction object that represents the intersection of ``self`` 

238 with ``others``. 

239 """ 

240 result = self.copy() 

241 for other in others: 

242 result.intersection_update(other) 

243 return result 

244 

245 def update_extract(self, data_id: DataCoordinate) -> None: 

246 """Update ``self`` to include all governor dimension values in the 

247 given data ID (in addition to those already in ``self``). 

248 

249 Parameters 

250 ---------- 

251 data_id : `DataCoordinate` 

252 Data ID from which governor dimension values should be extracted. 

253 Values for non-governor dimensions are ignored. 

254 """ 

255 for dimension in data_id.graph.governors: 

256 current = self._mapping.get(dimension) 

257 if current is not None: 

258 current.add(data_id[dimension]) 

259 

260 

261@dataclass 

262class CollectionSummary: 

263 """A summary of the datasets that can be found in a collection. 

264 """ 

265 

266 @classmethod 

267 def makeEmpty(cls, universe: DimensionUniverse) -> CollectionSummary: 

268 """Construct a `CollectionSummary` for a collection with no 

269 datasets. 

270 

271 Parameters 

272 ---------- 

273 universe : `DimensionUniverse` 

274 Object that manages all dimensions. 

275 

276 Returns 

277 ------- 

278 summary : `CollectionSummary` 

279 Summary object with no dataset types and no governor dimension 

280 values. 

281 """ 

282 return cls( 

283 datasetTypes=NamedValueSet(), 

284 dimensions=GovernorDimensionRestriction.makeEmpty(universe), 

285 ) 

286 

287 def copy(self) -> CollectionSummary: 

288 """Return a deep copy of this object. 

289 

290 Returns 

291 ------- 

292 copy : `CollectionSummary` 

293 A copy of ``self`` that can be modified without modifying ``self`` 

294 at all. 

295 """ 

296 return CollectionSummary(datasetTypes=self.datasetTypes.copy(), dimensions=self.dimensions.copy()) 

297 

298 def union(self, *others: CollectionSummary) -> CollectionSummary: 

299 """Construct a summary that contains all dataset types and governor 

300 dimension values in any of the inputs. 

301 

302 Parameters 

303 ---------- 

304 *others : `CollectionSummary` 

305 Restrictions to combine with ``self``. 

306 

307 Returns 

308 ------- 

309 unioned : `CollectionSummary` 

310 New summary object that represents the union of ``self`` with 

311 ``others``. 

312 """ 

313 if not others: 

314 return self 

315 datasetTypes = NamedValueSet(self.datasetTypes) 

316 datasetTypes.update(itertools.chain.from_iterable(o.datasetTypes for o in others)) 

317 dimensions = self.dimensions.union(*[o.dimensions for o in others]) 

318 return CollectionSummary(datasetTypes, dimensions) 

319 

320 def is_compatible_with( 

321 self, 

322 datasetType: DatasetType, 

323 restriction: GovernorDimensionRestriction, 

324 rejections: Optional[List[str]] = None, 

325 name: Optional[str] = None, 

326 ) -> bool: 

327 """Test whether the collection summarized by this object should be 

328 queried for a given dataset type and governor dimension values. 

329 

330 Parameters 

331 ---------- 

332 datasetType : `DatasetType` 

333 Dataset type being queried. If this collection has no instances of 

334 this dataset type (or its parent dataset type, if it is a 

335 component), `False` will always be returned. 

336 restriction : `GovernorDimensionRestriction` 

337 Restriction on the values governor dimensions can take in the 

338 query, usually from a WHERE expression. If this is disjoint with 

339 the data IDs actually present in the collection, `False` will be 

340 returned. 

341 rejections : `list` [ `str` ], optional 

342 If provided, a list that will be populated with a log- or 

343 exception-friendly message explaining why this dataset is 

344 incompatible with this collection when `False` is returned. 

345 name : `str`, optional 

346 Name of the collection this object summarizes, for use in messages 

347 appended to ``rejections``. Ignored if ``rejections`` is `None`. 

348 

349 Returns 

350 ------- 

351 compatible : `bool` 

352 `True` if the dataset query described by this summary and the given 

353 arguments might yield non-empty results; `False` if the result from 

354 such a query is definitely empty. 

355 """ 

356 parent = datasetType if not datasetType.isComponent() else datasetType.makeCompositeDatasetType() 

357 if parent not in self.datasetTypes: 

358 if rejections is not None: 

359 rejections.append(f"No datasets of type {parent.name} in collection {name!r}.") 

360 return False 

361 for governor in datasetType.dimensions.governors: 

362 if (values_in_self := self.dimensions.get(governor)) is not None: 

363 if (values_in_other := restriction.get(governor)) is not None: 

364 if values_in_self.isdisjoint(values_in_other): 

365 assert values_in_other, f"No valid values in restriction for dimension {governor}." 

366 if rejections is not None: 

367 rejections.append( 

368 f"No datasets with {governor.name} in {values_in_other} " 

369 f"in collection {name!r}." 

370 ) 

371 return False 

372 return True 

373 

374 datasetTypes: NamedValueSet[DatasetType] 

375 """Dataset types that may be present in the collection 

376 (`NamedValueSet` [ `DatasetType` ]). 

377 

378 A dataset type not in this set is definitely not in the collection, but 

379 the converse is not necessarily true. 

380 """ 

381 

382 dimensions: GovernorDimensionRestriction 

383 """Governor dimension values that may be present in the collection 

384 (`GovernorDimensionRestriction`). 

385 

386 A dimension value not in this restriction is definitely not in the 

387 collection, but the converse is not necessarily true. 

388 """