Coverage for python/lsst/daf/butler/core/utils.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "getObjectSize",
29 "immutable",
30 "IndexedTupleDict",
31 "iterable",
32 "NamedKeyDict",
33 "NamedValueSet",
34 "PrivateConstructorMeta",
35 "Singleton",
36 "slotValuesAreEqual",
37 "slotValuesToHash",
38 "stripIfNotNone",
39 "transactional",
40)
42import builtins
43import sys
44import functools
45from typing import (TypeVar, MutableMapping, Iterator, KeysView, ValuesView, ItemsView, Dict, Union,
46 MutableSet, Iterable, Mapping, Tuple)
47from types import MappingProxyType
49from lsst.utils import doImport
52def iterable(a):
53 """Make input iterable.
55 There are three cases, when the input is:
57 - iterable, but not a `str` or Mapping -> iterate over elements
58 (e.g. ``[i for i in a]``)
59 - a `str` -> return single element iterable (e.g. ``[a]``)
60 - a Mapping -> return single element iterable
61 - not iterable -> return single elment iterable (e.g. ``[a]``).
63 Parameters
64 ----------
65 a : iterable or `str` or not iterable
66 Argument to be converted to an iterable.
68 Returns
69 -------
70 i : `generator`
71 Iterable version of the input value.
72 """
73 if isinstance(a, str):
74 yield a
75 return
76 if isinstance(a, Mapping):
77 yield a
78 return
79 try:
80 yield from a
81 except Exception:
82 yield a
85def allSlots(self):
86 """
87 Return combined ``__slots__`` for all classes in objects mro.
89 Parameters
90 ----------
91 self : `object`
92 Instance to be inspected.
94 Returns
95 -------
96 slots : `itertools.chain`
97 All the slots as an iterable.
98 """
99 from itertools import chain
100 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
103def slotValuesAreEqual(self, other):
104 """
105 Test for equality by the contents of all slots, including those of its
106 parents.
108 Parameters
109 ----------
110 self : `object`
111 Reference instance.
112 other : `object`
113 Comparison instance.
115 Returns
116 -------
117 equal : `bool`
118 Returns True if all the slots are equal in both arguments.
119 """
120 return all((getattr(self, slot) == getattr(other, slot) for slot in allSlots(self)))
123def slotValuesToHash(self):
124 """
125 Generate a hash from slot values.
127 Parameters
128 ----------
129 self : `object`
130 Instance to be hashed.
132 Returns
133 -------
134 h : `int`
135 Hashed value generated from the slot values.
136 """
137 return hash(tuple(getattr(self, slot) for slot in allSlots(self)))
140def getFullTypeName(cls):
141 """Return full type name of the supplied entity.
143 Parameters
144 ----------
145 cls : `type` or `object`
146 Entity from which to obtain the full name. Can be an instance
147 or a `type`.
149 Returns
150 -------
151 name : `str`
152 Full name of type.
154 Notes
155 -----
156 Builtins are returned without the ``builtins`` specifier included. This
157 allows `str` to be returned as "str" rather than "builtins.str".
158 """
159 # If we have an instance we need to convert to a type
160 if not hasattr(cls, "__qualname__"): 160 ↛ 161line 160 didn't jump to line 161, because the condition on line 160 was never true
161 cls = type(cls)
162 if hasattr(builtins, cls.__qualname__): 162 ↛ 164line 162 didn't jump to line 164, because the condition on line 162 was never true
163 # Special case builtins such as str and dict
164 return cls.__qualname__
165 return cls.__module__ + "." + cls.__qualname__
168def getClassOf(typeOrName):
169 """Given the type name or a type, return the python type.
171 If a type name is given, an attempt will be made to import the type.
173 Parameters
174 ----------
175 typeOrName : `str` or Python class
176 A string describing the Python class to load or a Python type.
178 Returns
179 -------
180 type_ : `type`
181 Directly returns the Python type if a type was provided, else
182 tries to import the given string and returns the resulting type.
184 Notes
185 -----
186 This is a thin wrapper around `~lsst.utils.doImport`.
187 """
188 if isinstance(typeOrName, str):
189 cls = doImport(typeOrName)
190 else:
191 cls = typeOrName
192 return cls
195def getInstanceOf(typeOrName, *args, **kwargs):
196 """Given the type name or a type, instantiate an object of that type.
198 If a type name is given, an attempt will be made to import the type.
200 Parameters
201 ----------
202 typeOrName : `str` or Python class
203 A string describing the Python class to load or a Python type.
204 args : `tuple`
205 Positional arguments to use pass to the object constructor.
206 kwargs : `dict`
207 Keyword arguments to pass to object constructor.
209 Returns
210 -------
211 instance : `object`
212 Instance of the requested type, instantiated with the provided
213 parameters.
214 """
215 cls = getClassOf(typeOrName)
216 return cls(*args, **kwargs)
219class Singleton(type):
220 """Metaclass to convert a class to a Singleton.
222 If this metaclass is used the constructor for the singleton class must
223 take no arguments. This is because a singleton class will only accept
224 the arguments the first time an instance is instantiated.
225 Therefore since you do not know if the constructor has been called yet it
226 is safer to always call it with no arguments and then call a method to
227 adjust state of the singleton.
228 """
230 _instances = {}
232 def __call__(cls):
233 if cls not in cls._instances:
234 cls._instances[cls] = super(Singleton, cls).__call__()
235 return cls._instances[cls]
238def transactional(func):
239 """Decorator that wraps a method and makes it transactional.
241 This depends on the class also defining a `transaction` method
242 that takes no arguments and acts as a context manager.
243 """
244 @functools.wraps(func)
245 def inner(self, *args, **kwargs):
246 with self.transaction():
247 return func(self, *args, **kwargs)
248 return inner
251def getObjectSize(obj, seen=None):
252 """Recursively finds size of objects.
254 Only works well for pure python objects. For example it does not work for
255 ``Exposure`` objects where all the content is behind getter methods.
257 Parameters
258 ----------
259 obj : `object`
260 Instance for which size is to be calculated.
261 seen : `set`, optional
262 Used internally to keep track of objects already sized during
263 recursion.
265 Returns
266 -------
267 size : `int`
268 Size in bytes.
270 See Also
271 --------
272 sys.getsizeof
274 Notes
275 -----
276 See https://goshippo.com/blog/measure-real-size-any-python-object/
277 """
278 size = sys.getsizeof(obj)
279 if seen is None:
280 seen = set()
281 obj_id = id(obj)
282 if obj_id in seen:
283 return 0
284 # Important mark as seen *before* entering recursion to gracefully handle
285 # self-referential objects
286 seen.add(obj_id)
287 if isinstance(obj, dict):
288 size += sum([getObjectSize(v, seen) for v in obj.values()])
289 size += sum([getObjectSize(k, seen) for k in obj.keys()])
290 elif hasattr(obj, "__dict__"):
291 size += getObjectSize(obj.__dict__, seen)
292 elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes, bytearray)):
293 size += sum([getObjectSize(i, seen) for i in obj])
295 return size
298def stripIfNotNone(s):
299 """Strip leading and trailing whitespace if the given object is not None.
301 Parameters
302 ----------
303 s : `str`, optional
304 Input string.
306 Returns
307 -------
308 r : `str` or `None`
309 A string with leading and trailing whitespace stripped if `s` is not
310 `None`, or `None` if `s` is `None`.
311 """
312 if s is not None:
313 s = s.strip()
314 return s
317class PrivateConstructorMeta(type):
318 """A metaclass that disables regular construction syntax.
320 A class that uses PrivateConstructorMeta may have an ``__init__`` and/or
321 ``__new__`` method, but these can't be invoked by "calling" the class
322 (that will always raise `TypeError`). Instead, such classes can be called
323 by calling the metaclass-provided `_construct` class method with the same
324 arguments.
326 As is usual in Python, there are no actual prohibitions on what code can
327 call `_construct`; the purpose of this metaclass is just to prevent
328 instances from being created normally when that can't do what users would
329 expect.
331 ..note::
333 Classes that inherit from PrivateConstructorMeta also inherit
334 the hidden-constructor behavior. If you just want to disable
335 construction of the base class, `abc.ABCMeta` may be a better
336 option.
338 Examples
339 --------
340 Given this class definition::
341 class Hidden(metaclass=PrivateConstructorMeta):
343 def __init__(self, a, b):
344 self.a = a
345 self.b = b
347 This doesn't work:
349 >>> instance = Hidden(a=1, b="two")
350 TypeError: Hidden objects cannot be constructed directly.
352 But this does:
354 >>> instance = Hidden._construct(a=1, b="two")
356 """
358 def __call__(cls, *args, **kwds):
359 """Disabled class construction interface; always raises `TypeError.`
360 """
361 raise TypeError(f"{cls.__name__} objects cannot be constructed directly.")
363 def _construct(cls, *args, **kwds):
364 """Private class construction interface.
366 All arguments are forwarded to ``__init__`` and/or ``__new__``
367 in the usual way.
368 """
369 return type.__call__(cls, *args, **kwds)
372K = TypeVar("K")
373V = TypeVar("V")
376class NamedKeyDict(MutableMapping[K, V]):
377 """A dictionary wrapper that require keys to have a ``.name`` attribute,
378 and permits lookups using either key objects or their names.
380 Names can be used in place of keys when updating existing items, but not
381 when adding new items.
383 It is assumed (but asserted) that all name equality is equivalent to key
384 equality, either because the key objects define equality this way, or
385 because different objects with the same name are never included in the same
386 dictionary.
388 Parameters
389 ----------
390 args
391 All positional constructor arguments are forwarded directly to `dict`.
392 Keyword arguments are not accepted, because plain strings are not valid
393 keys for `NamedKeyDict`.
395 Raises
396 ------
397 AttributeError
398 Raised when an attempt is made to add an object with no ``.name``
399 attribute to the dictionary.
400 AssertionError
401 Raised when multiple keys have the same name.
402 """
404 __slots__ = ("_dict", "_names",)
406 def __init__(self, *args):
407 self._dict = dict(*args)
408 self._names = {key.name: key for key in self._dict}
409 assert len(self._names) == len(self._dict), "Duplicate names in keys."
411 @property
412 def names(self) -> KeysView[str]:
413 """The set of names associated with the keys, in the same order
414 (`~collections.abc.KeysView`).
415 """
416 return self._names.keys()
418 def byName(self) -> Dict[str, V]:
419 """Return a `dict` with names as keys and the same values as ``self``.
420 """
421 return dict(zip(self._names.keys(), self._dict.values()))
423 def __len__(self) -> int:
424 return len(self._dict)
426 def __iter__(self) -> Iterator[K]:
427 return iter(self._dict)
429 def __str__(self) -> str:
430 return "{{{}}}".format(", ".join(f"{str(k)}: {str(v)}" for k, v in self.items()))
432 def __repr__(self) -> str:
433 return "NamedKeyDict({{{}}})".format(", ".join(f"{repr(k)}: {repr(v)}" for k, v in self.items()))
435 def __getitem__(self, key: Union[str, K]) -> V:
436 if hasattr(key, "name"):
437 return self._dict[key]
438 else:
439 return self._dict[self._names[key]]
441 def __setitem__(self, key: Union[str, K], value: V):
442 if hasattr(key, "name"):
443 assert self._names.get(key.name, key) == key, "Name is already associated with a different key."
444 self._dict[key] = value
445 self._names[key.name] = key
446 else:
447 self._dict[self._names[key]] = value
449 def __delitem__(self, key: Union[str, K]):
450 if hasattr(key, "name"):
451 del self._dict[key]
452 del self._names[key.name]
453 else:
454 del self._dict[self._names[key]]
455 del self._names[key]
457 def keys(self) -> KeysView[K]:
458 return self._dict.keys()
460 def values(self) -> ValuesView[V]:
461 return self._dict.values()
463 def items(self) -> ItemsView[K, V]:
464 return self._dict.items()
466 def copy(self) -> NamedKeyDict[K, V]:
467 result = NamedKeyDict.__new__(NamedKeyDict)
468 result._dict = dict(self._dict)
469 result._names = dict(self._names)
470 return result
472 def freeze(self):
473 """Disable all mutators, effectively transforming ``self`` into
474 an immutable mapping.
475 """
476 if not isinstance(self._dict, MappingProxyType):
477 self._dict = MappingProxyType(self._dict)
480T = TypeVar("T")
483class NamedValueSet(MutableSet[T]):
484 """A custom mutable set class that requires elements to have a ``.name``
485 attribute, which can then be used as keys in `dict`-like lookup.
487 Names and elements can both be used with the ``in`` and ``del``
488 operators, `remove`, and `discard`. Names (but not elements)
489 can be used with ``[]``-based element retrieval (not assignment)
490 and the `get` method. `pop` can be used in either its `MutableSet`
491 form (no arguments; an arbitrary element is returned) or its
492 `MutableMapping` form (one or two arguments for the name and
493 optional default value, respectively).
495 Parameters
496 ----------
497 elements : `iterable`
498 Iterable over elements to include in the set.
500 Raises
501 ------
502 AttributeError
503 Raised if one or more elements do not have a ``.name`` attribute.
505 Notes
506 -----
507 Iteration order is guaranteed to be the same as insertion order (with
508 the same general behavior as `dict` ordering).
509 Like `dicts`, sets with the same elements will compare as equal even if
510 their iterator order is not the same.
511 """
513 __slots__ = ("_dict",)
515 def __init__(self, elements: Iterable[T] = ()):
516 self._dict = {element.name: element for element in elements}
518 @property
519 def names(self) -> KeysView[str]:
520 """The set of element names, in the same order
521 (`~collections.abc.KeysView`).
522 """
523 return self._dict.keys()
525 def asDict(self) -> Mapping[str, T]:
526 """Return a mapping view with names as keys.
528 Returns
529 -------
530 dict : `Mapping`
531 A dictionary-like view with ``values() == self``.
532 """
533 return self._dict
535 def __contains__(self, key: Union[str, T]) -> bool:
536 return getattr(key, "name", key) in self._dict
538 def __len__(self) -> int:
539 return len(self._dict)
541 def __iter__(self) -> Iterator[T]:
542 return iter(self._dict.values())
544 def __str__(self) -> str:
545 return "{{{}}}".format(", ".join(str(element) for element in self))
547 def __repr__(self) -> str:
548 return "NamedValueSet({{{}}})".format(", ".join(repr(element) for element in self))
550 def __eq__(self, other):
551 try:
552 return self._dict.keys() == other._dict.keys()
553 except AttributeError:
554 return NotImplemented
556 def __hash__(self):
557 return hash(frozenset(self._dict.keys()))
559 # As per Set's docs, overriding just __le__ and __ge__ for performance will
560 # cover the other comparisons, too.
562 def __le__(self, other: NamedValueSet[T]) -> bool:
563 try:
564 return self._dict.keys() <= other._dict.keys()
565 except AttributeError:
566 return NotImplemented
568 def __ge__(self, other: NamedValueSet[T]) -> bool:
569 try:
570 return self._dict.keys() >= other._dict.keys()
571 except AttributeError:
572 return NotImplemented
574 def issubset(self, other):
575 return self <= other
577 def issuperset(self, other):
578 return self >= other
580 def __getitem__(self, name: str) -> T:
581 return self._dict[name]
583 def get(self, name: str, default=None):
584 """Return the element with the given name, or ``default`` if
585 no such element is present.
586 """
587 return self._dict.get(name, default)
589 def __delitem__(self, name: str):
590 del self._dict[name]
592 def add(self, element: T):
593 """Add an element to the set.
595 Raises
596 ------
597 AttributeError
598 Raised if the element does not have a ``.name`` attribute.
599 """
600 self._dict[element.name] = element
602 def remove(self, element: Union[str, T]):
603 """Remove an element from the set.
605 Parameters
606 ----------
607 element : `object` or `str`
608 Element to remove or the string name thereof. Assumed to be an
609 element if it has a ``.name`` attribute.
611 Raises
612 ------
613 KeyError
614 Raised if an element with the given name does not exist.
615 """
616 del self._dict[getattr(element, "name", element)]
618 def discard(self, element: Union[str, T]):
619 """Remove an element from the set if it exists.
621 Does nothing if no matching element is present.
623 Parameters
624 ----------
625 element : `object` or `str`
626 Element to remove or the string name thereof. Assumed to be an
627 element if it has a ``.name`` attribute.
628 """
629 try:
630 self.remove(element)
631 except KeyError:
632 pass
634 def pop(self, *args):
635 """Remove and return an element from the set.
637 Parameters
638 ----------
639 name : `str`, optional
640 Name of the element to remove and return. Must be passed
641 positionally. If not provided, an arbitrary element is
642 removed and returned.
643 default : `object`, optional
644 Value to return if ``name`` is provided but no such element
645 exists.
647 Raises
648 ------
649 KeyError
650 Raised if ``name`` is provided but ``default`` is not, and no
651 matching element exists.
652 """
653 if not args:
654 return super().pop()
655 else:
656 return self._dict.pop(*args)
658 def copy(self) -> NamedValueSet[T]:
659 result = NamedValueSet.__new__(NamedValueSet)
660 result._dict = dict(self._dict)
661 return result
663 def freeze(self):
664 """Disable all mutators, effectively transforming ``self`` into
665 an immutable set.
666 """
667 if not isinstance(self._dict, MappingProxyType):
668 self._dict = MappingProxyType(self._dict)
671class IndexedTupleDict(Mapping[K, V]):
672 """An immutable mapping that combines a tuple of values with a (possibly
673 shared) mapping from key to tuple index.
675 Parameters
676 ----------
677 indices: `~collections.abc.Mapping`
678 Mapping from key to integer index in the values tuple. This mapping
679 is used as-is, not copied or converted to a true `dict`, which means
680 that the caller must guarantee that it will not be modified by other
681 (shared) owners in the future. If it is a `NamedKeyDict`, both names
682 and key instances will be usable as keys in the `IndexedTupleDict`.
683 The caller is also responsible for guaranteeing that the indices in
684 the mapping are all valid for the given tuple.
685 values: `tuple`
686 Tuple of values for the dictionary. The caller is responsible for
687 guaranteeing that this has the same number of elements as ``indices``.
688 """
690 __slots__ = ("_indices", "_values")
692 def __new__(cls, indices: Mapping[K, int], values: Tuple[V, ...]):
693 self = super().__new__(cls)
694 assert len(indices) == len(values)
695 self._indices = indices
696 self._values = values
697 return self
699 def __getitem__(self, key: K) -> V:
700 return self._values[self._indices[key]]
702 def __iter__(self) -> Iterator[K]:
703 return iter(self._indices)
705 def __len__(self) -> int:
706 return len(self._indices)
708 def __str__(self) -> str:
709 return "{{{}}}".format(", ".join(f"{str(k)}: {str(v)}" for k, v in self.items()))
711 def __repr__(self) -> str:
712 return "IndexedTupleDict({{{}}})".format(", ".join(f"{repr(k)}: {repr(v)}" for k, v in self.items()))
714 def __contains__(self, key: K) -> bool:
715 return key in self._indices
717 def keys(self) -> KeysView[K]:
718 return self._indices.keys()
720 def values(self) -> Tuple[V, ...]:
721 return self._values
723 def __getnewargs__(self) -> tuple:
724 return (self._indices, self._values)
726 def __getstate__(self) -> dict: # noqa: N807
727 # Disable default state-setting when unpickled.
728 return {}
730 def __setstate__(self, state): # noqa: N807
731 # Disable default state-setting when copied.
732 # Sadly what works for pickle doesn't work for copy.
733 assert not state
735 # Let Mapping base class provide items(); we can't do it any more
736 # efficiently ourselves.
739def immutable(cls):
740 """A class decorator that simulates a simple form of immutability for
741 the decorated class.
743 A class decorated as `immutable` may only set each of its attributes once
744 (by convention, in ``__new__``); any attempts to set an already-set
745 attribute will raise `AttributeError`.
747 Because this behavior interferes with the default implementation for
748 the ``pickle`` and ``copy`` modules, `immutable` provides implementations
749 of ``__getstate__`` and ``__setstate__`` that override this behavior.
750 Immutable classes can them implement pickle/copy via ``__getnewargs__``
751 only (other approaches such as ``__reduce__`` and ``__deepcopy__`` may
752 also be used).
753 """
754 def __setattr__(self, name, value): # noqa: N807
755 if hasattr(self, name):
756 raise AttributeError(f"{cls.__name__} instances are immutable.")
757 object.__setattr__(self, name, value)
758 cls.__setattr__ = __setattr__
760 def __getstate__(self) -> dict: # noqa: N807
761 # Disable default state-setting when unpickled.
762 return {}
763 cls.__getstate__ = __getstate__
765 def __setstate__(self, state): # noqa: N807
766 # Disable default state-setting when copied.
767 # Sadly what works for pickle doesn't work for copy.
768 assert not state
769 cls.__setstate__ = __setstate__
770 return cls