Coverage for python/lsst/daf/butler/registry/summaries.py: 29%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "CollectionSummary",
25 "GovernorDimensionRestriction",
26)
28from dataclasses import dataclass
29import itertools
30from typing import (
31 AbstractSet,
32 Any,
33 ItemsView,
34 Iterable,
35 Iterator,
36 List,
37 Mapping,
38 Optional,
39 Set,
40 Union,
41 ValuesView,
42)
43from lsst.utils.iteration import ensure_iterable
45from ..core import (
46 DataCoordinate,
47 DatasetType,
48 DimensionUniverse,
49 GovernorDimension,
50 NamedKeyDict,
51 NamedKeyMapping,
52 NamedValueAbstractSet,
53 NamedValueSet,
54)
57class GovernorDimensionRestriction(NamedKeyMapping[GovernorDimension, AbstractSet[str]]):
58 """A custom mapping that represents a restriction on the values one or
59 more governor dimensions may take in some context.
61 Parameters
62 ----------
63 mapping : `NamedKeyDict` [ `GovernorDimension`, `Set` [ `str` ]]
64 Mapping from governor dimension to the values it may take. Dimensions
65 not present in the mapping are not constrained at all.
66 """
67 def __init__(self, mapping: NamedKeyDict[GovernorDimension, Set[str]]):
68 self._mapping = mapping
70 @classmethod
71 def makeEmpty(cls, universe: DimensionUniverse) -> GovernorDimensionRestriction:
72 """Construct a `GovernorDimensionRestriction` that allows no values
73 for any governor dimension in the given `DimensionUniverse`.
75 Parameters
76 ----------
77 universe : `DimensionUniverse`
78 Object that manages all dimensions.
80 Returns
81 -------
82 restriction : `GovernorDimensionRestriction`
83 Restriction instance that maps all governor dimensions to an empty
84 set.
85 """
86 return cls(NamedKeyDict((k, set()) for k in universe.getGovernorDimensions()))
88 @classmethod
89 def makeFull(cls) -> GovernorDimensionRestriction:
90 """Construct a `GovernorDimensionRestriction` that allows any value
91 for any governor dimension.
93 Returns
94 -------
95 restriction : `GovernorDimensionRestriction`
96 Restriction instance that contains no keys, and hence contains
97 allows any value for any governor dimension.
98 """
99 return cls(NamedKeyDict())
101 def __eq__(self, other: Any) -> bool:
102 if not isinstance(other, GovernorDimensionRestriction):
103 return False
104 return self._mapping == other._mapping
106 def __str__(self) -> str:
107 return "({})".format(
108 ", ".join(f"{dimension.name}: {values}" for dimension, values in self._mapping.items())
109 )
111 def __repr__(self) -> str:
112 return "GovernorDimensionRestriction({})".format(
113 ", ".join(f"{dimension.name}={values}" for dimension, values in self._mapping.items())
114 )
116 def __iter__(self) -> Iterator[GovernorDimension]:
117 return iter(self._mapping)
119 def __len__(self) -> int:
120 return len(self._mapping)
122 @property
123 def names(self) -> AbstractSet[str]:
124 # Docstring inherited.
125 return self._mapping.names
127 def keys(self) -> NamedValueAbstractSet[GovernorDimension]:
128 return self._mapping.keys()
130 def values(self) -> ValuesView[AbstractSet[str]]:
131 return self._mapping.values()
133 def items(self) -> ItemsView[GovernorDimension, AbstractSet[str]]:
134 return self._mapping.items()
136 def __getitem__(self, key: Union[str, GovernorDimension]) -> AbstractSet[str]:
137 return self._mapping[key]
139 def copy(self) -> GovernorDimensionRestriction:
140 """Return a deep copy of this object.
142 Returns
143 -------
144 copy : `GovernorDimensionRestriction`
145 A copy of ``self`` that can be modified without modifying ``self``
146 at all.
147 """
148 return GovernorDimensionRestriction(NamedKeyDict((k, set(v)) for k, v in self.items()))
150 def add(self, dimension: GovernorDimension, value: str) -> None:
151 """Add a single dimension value to the restriction.
153 Parameters
154 ----------
155 dimension : `GovernorDimension`
156 Dimension to update.
157 value : `str`
158 Value to allow for this dimension.
159 """
160 current = self._mapping.get(dimension)
161 if current is not None:
162 current.add(value)
164 def update(self, other: Mapping[GovernorDimension, Union[str, Iterable[str]]]) -> None:
165 """Update ``self`` to include all dimension values in either ``self``
166 or ``other``.
168 Parameters
169 ----------
170 other : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ]
171 Mapping to union into ``self``. This may be another
172 `GovernorDimensionRestriction` or any other mapping from dimension
173 to `str` or iterable of `str`.
174 """
175 for dimension in (self.keys() - other.keys()):
176 self._mapping.pop(dimension, None)
177 for dimension in (self.keys() & other.keys()):
178 self._mapping[dimension].update(ensure_iterable(other[dimension]))
179 # Dimensions that are in 'other' but not in 'self' are ignored, because
180 # 'self' says they are already unconstrained.
182 def union(self, *others: Mapping[GovernorDimension, Union[str, Iterable[str]]]
183 ) -> GovernorDimensionRestriction:
184 """Construct a restriction that permits any values permitted by any of
185 the input restrictions.
187 Parameters
188 ----------
189 *others : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ]
190 Mappings to union into ``self``. These may be other
191 `GovernorDimensionRestriction` instances or any other kind of
192 mapping from dimension to `str` or iterable of `str`.
194 Returns
195 -------
196 unioned : `GovernorDimensionRestriction`
197 New restriction object that represents the union of ``self`` with
198 ``others``.
199 """
200 result = self.copy()
201 for other in others:
202 result.update(other)
203 return result
205 def intersection_update(self, other: Mapping[GovernorDimension, Union[str, Iterable[str]]]) -> None:
206 """Update ``self`` to include only dimension values in both ``self``
207 and ``other``.
209 Parameters
210 ----------
211 other : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ]
212 Mapping to intersect into ``self``. This may be another
213 `GovernorDimensionRestriction` or any other mapping from dimension
214 to `str` or iterable of `str`.
215 """
216 for dimension, values in other.items():
217 new_values = set(ensure_iterable(values))
218 # Yes, this will often result in a (no-op) self-intersection on the
219 # inner set, but this is easier to read (and obviously more or less
220 # efficient) than adding a check to avoid it.
221 self._mapping.setdefault(dimension, new_values).intersection_update(new_values)
223 def intersection(self, *others: Mapping[GovernorDimension, Union[str, Iterable[str]]]
224 ) -> GovernorDimensionRestriction:
225 """Construct a restriction that permits only values permitted by all of
226 the input restrictions.
228 Parameters
229 ----------
230 *others : `Mapping` [ `Dimension`, `str` or `Iterable` [ `str` ] ]
231 Mappings to intersect with ``self``. These may be other
232 `GovernorDimensionRestriction` instances or any other kind of
233 mapping from dimension to `str` or iterable of `str`.
234 Returns
235 -------
236 intersection : `GovernorDimensionRestriction`
237 New restriction object that represents the intersection of ``self``
238 with ``others``.
239 """
240 result = self.copy()
241 for other in others:
242 result.intersection_update(other)
243 return result
245 def update_extract(self, data_id: DataCoordinate) -> None:
246 """Update ``self`` to include all governor dimension values in the
247 given data ID (in addition to those already in ``self``).
249 Parameters
250 ----------
251 data_id : `DataCoordinate`
252 Data ID from which governor dimension values should be extracted.
253 Values for non-governor dimensions are ignored.
254 """
255 for dimension in data_id.graph.governors:
256 current = self._mapping.get(dimension)
257 if current is not None:
258 current.add(data_id[dimension])
261@dataclass
262class CollectionSummary:
263 """A summary of the datasets that can be found in a collection.
264 """
266 @classmethod
267 def makeEmpty(cls, universe: DimensionUniverse) -> CollectionSummary:
268 """Construct a `CollectionSummary` for a collection with no
269 datasets.
271 Parameters
272 ----------
273 universe : `DimensionUniverse`
274 Object that manages all dimensions.
276 Returns
277 -------
278 summary : `CollectionSummary`
279 Summary object with no dataset types and no governor dimension
280 values.
281 """
282 return cls(
283 datasetTypes=NamedValueSet(),
284 dimensions=GovernorDimensionRestriction.makeEmpty(universe),
285 )
287 def copy(self) -> CollectionSummary:
288 """Return a deep copy of this object.
290 Returns
291 -------
292 copy : `CollectionSummary`
293 A copy of ``self`` that can be modified without modifying ``self``
294 at all.
295 """
296 return CollectionSummary(datasetTypes=self.datasetTypes.copy(), dimensions=self.dimensions.copy())
298 def union(self, *others: CollectionSummary) -> CollectionSummary:
299 """Construct a summary that contains all dataset types and governor
300 dimension values in any of the inputs.
302 Parameters
303 ----------
304 *others : `CollectionSummary`
305 Restrictions to combine with ``self``.
307 Returns
308 -------
309 unioned : `CollectionSummary`
310 New summary object that represents the union of ``self`` with
311 ``others``.
312 """
313 if not others:
314 return self
315 datasetTypes = NamedValueSet(self.datasetTypes)
316 datasetTypes.update(itertools.chain.from_iterable(o.datasetTypes for o in others))
317 dimensions = self.dimensions.union(*[o.dimensions for o in others])
318 return CollectionSummary(datasetTypes, dimensions)
320 def is_compatible_with(
321 self,
322 datasetType: DatasetType,
323 restriction: GovernorDimensionRestriction,
324 rejections: Optional[List[str]] = None,
325 name: Optional[str] = None,
326 ) -> bool:
327 """Test whether the collection summarized by this object should be
328 queried for a given dataset type and governor dimension values.
330 Parameters
331 ----------
332 datasetType : `DatasetType`
333 Dataset type being queried. If this collection has no instances of
334 this dataset type (or its parent dataset type, if it is a
335 component), `False` will always be returned.
336 restriction : `GovernorDimensionRestriction`
337 Restriction on the values governor dimensions can take in the
338 query, usually from a WHERE expression. If this is disjoint with
339 the data IDs actually present in the collection, `False` will be
340 returned.
341 rejections : `list` [ `str` ], optional
342 If provided, a list that will be populated with a log- or
343 exception-friendly message explaining why this dataset is
344 incompatible with this collection when `False` is returned.
345 name : `str`, optional
346 Name of the collection this object summarizes, for use in messages
347 appended to ``rejections``. Ignored if ``rejections`` is `None`.
349 Returns
350 -------
351 compatible : `bool`
352 `True` if the dataset query described by this summary and the given
353 arguments might yield non-empty results; `False` if the result from
354 such a query is definitely empty.
355 """
356 parent = datasetType if not datasetType.isComponent() else datasetType.makeCompositeDatasetType()
357 if parent not in self.datasetTypes:
358 if rejections is not None:
359 rejections.append(f"No datasets of type {parent.name} in collection {name!r}.")
360 return False
361 for governor in datasetType.dimensions.governors:
362 if (values_in_self := self.dimensions.get(governor)) is not None:
363 if (values_in_other := restriction.get(governor)) is not None:
364 if values_in_self.isdisjoint(values_in_other):
365 assert values_in_other, f"No valid values in restriction for dimension {governor}."
366 if rejections is not None:
367 rejections.append(
368 f"No datasets with {governor.name} in {values_in_other} "
369 f"in collection {name!r}."
370 )
371 return False
372 return True
374 datasetTypes: NamedValueSet[DatasetType]
375 """Dataset types that may be present in the collection
376 (`NamedValueSet` [ `DatasetType` ]).
378 A dataset type not in this set is definitely not in the collection, but
379 the converse is not necessarily true.
380 """
382 dimensions: GovernorDimensionRestriction
383 """Governor dimension values that may be present in the collection
384 (`GovernorDimensionRestriction`).
386 A dimension value not in this restriction is definitely not in the
387 collection, but the converse is not necessarily true.
388 """