Coverage for python/lsst/daf/butler/core/utils.py : 41%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "iterable",
30 "safeMakeDir",
31 "Singleton",
32 "stripIfNotNone",
33 "transactional",
34)
36import errno
37import os
38import builtins
39import fnmatch
40import functools
41import logging
42import re
43from typing import (
44 Any,
45 Callable,
46 Dict,
47 Iterable,
48 Iterator,
49 List,
50 Mapping,
51 Optional,
52 Pattern,
53 Type,
54 TypeVar,
55 TYPE_CHECKING,
56 Union,
57)
59from lsst.utils import doImport
61if TYPE_CHECKING: 61 ↛ 62line 61 didn't jump to line 62, because the condition on line 61 was never true
62 from ..registry.wildcards import Ellipsis, EllipsisType
65_LOG = logging.getLogger(__name__)
68def safeMakeDir(directory: str) -> None:
69 """Make a directory in a manner avoiding race conditions"""
70 if directory != "" and not os.path.exists(directory):
71 try:
72 os.makedirs(directory)
73 except OSError as e:
74 # Don't fail if directory exists due to race
75 if e.errno != errno.EEXIST:
76 raise e
79def iterable(a: Any) -> Iterable[Any]:
80 """Make input iterable.
82 There are three cases, when the input is:
84 - iterable, but not a `str` or Mapping -> iterate over elements
85 (e.g. ``[i for i in a]``)
86 - a `str` -> return single element iterable (e.g. ``[a]``)
87 - a Mapping -> return single element iterable
88 - not iterable -> return single element iterable (e.g. ``[a]``).
90 Parameters
91 ----------
92 a : iterable or `str` or not iterable
93 Argument to be converted to an iterable.
95 Returns
96 -------
97 i : `generator`
98 Iterable version of the input value.
99 """
100 if isinstance(a, str):
101 yield a
102 return
103 if isinstance(a, Mapping):
104 yield a
105 return
106 try:
107 yield from a
108 except Exception:
109 yield a
112def allSlots(self: Any) -> Iterator[str]:
113 """
114 Return combined ``__slots__`` for all classes in objects mro.
116 Parameters
117 ----------
118 self : `object`
119 Instance to be inspected.
121 Returns
122 -------
123 slots : `itertools.chain`
124 All the slots as an iterable.
125 """
126 from itertools import chain
127 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
130def getFullTypeName(cls: Any) -> str:
131 """Return full type name of the supplied entity.
133 Parameters
134 ----------
135 cls : `type` or `object`
136 Entity from which to obtain the full name. Can be an instance
137 or a `type`.
139 Returns
140 -------
141 name : `str`
142 Full name of type.
144 Notes
145 -----
146 Builtins are returned without the ``builtins`` specifier included. This
147 allows `str` to be returned as "str" rather than "builtins.str". Any
148 parts of the path that start with a leading underscore are removed
149 on the assumption that they are an implementation detail and the
150 entity will be hoisted into the parent namespace.
151 """
152 # If we have an instance we need to convert to a type
153 if not hasattr(cls, "__qualname__"): 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 cls = type(cls)
155 if hasattr(builtins, cls.__qualname__):
156 # Special case builtins such as str and dict
157 return cls.__qualname__
159 real_name = cls.__module__ + "." + cls.__qualname__
161 # Remove components with leading underscores
162 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
164 # Consistency check
165 if real_name != cleaned_name: 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true
166 try:
167 test = doImport(cleaned_name)
168 except Exception:
169 # Could not import anything so return the real name
170 return real_name
172 # The thing we imported should match the class we started with
173 # despite the clean up. If it does not we return the real name
174 if test is not cls:
175 return real_name
177 return cleaned_name
180def getClassOf(typeOrName: Union[Type, str]) -> Type:
181 """Given the type name or a type, return the python type.
183 If a type name is given, an attempt will be made to import the type.
185 Parameters
186 ----------
187 typeOrName : `str` or Python class
188 A string describing the Python class to load or a Python type.
190 Returns
191 -------
192 type_ : `type`
193 Directly returns the Python type if a type was provided, else
194 tries to import the given string and returns the resulting type.
196 Notes
197 -----
198 This is a thin wrapper around `~lsst.utils.doImport`.
199 """
200 if isinstance(typeOrName, str):
201 cls = doImport(typeOrName)
202 else:
203 cls = typeOrName
204 return cls
207def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
208 """Given the type name or a type, instantiate an object of that type.
210 If a type name is given, an attempt will be made to import the type.
212 Parameters
213 ----------
214 typeOrName : `str` or Python class
215 A string describing the Python class to load or a Python type.
216 args : `tuple`
217 Positional arguments to use pass to the object constructor.
218 kwargs : `dict`
219 Keyword arguments to pass to object constructor.
221 Returns
222 -------
223 instance : `object`
224 Instance of the requested type, instantiated with the provided
225 parameters.
226 """
227 cls = getClassOf(typeOrName)
228 return cls(*args, **kwargs)
231class Singleton(type):
232 """Metaclass to convert a class to a Singleton.
234 If this metaclass is used the constructor for the singleton class must
235 take no arguments. This is because a singleton class will only accept
236 the arguments the first time an instance is instantiated.
237 Therefore since you do not know if the constructor has been called yet it
238 is safer to always call it with no arguments and then call a method to
239 adjust state of the singleton.
240 """
242 _instances: Dict[Type, Any] = {}
244 # Signature is intentionally not substitutable for type.__call__ (no *args,
245 # **kwargs) to require classes that use this metaclass to have no
246 # constructor arguments.
247 def __call__(cls) -> Any: # type: ignore
248 if cls not in cls._instances:
249 cls._instances[cls] = super(Singleton, cls).__call__()
250 return cls._instances[cls]
253F = TypeVar("F", bound=Callable)
256def transactional(func: F) -> F:
257 """Decorator that wraps a method and makes it transactional.
259 This depends on the class also defining a `transaction` method
260 that takes no arguments and acts as a context manager.
261 """
262 @functools.wraps(func)
263 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
264 with self.transaction():
265 return func(self, *args, **kwargs)
266 return inner # type: ignore
269def stripIfNotNone(s: Optional[str]) -> Optional[str]:
270 """Strip leading and trailing whitespace if the given object is not None.
272 Parameters
273 ----------
274 s : `str`, optional
275 Input string.
277 Returns
278 -------
279 r : `str` or `None`
280 A string with leading and trailing whitespace stripped if `s` is not
281 `None`, or `None` if `s` is `None`.
282 """
283 if s is not None:
284 s = s.strip()
285 return s
288_T = TypeVar("_T", bound="Type")
291def immutable(cls: _T) -> _T:
292 """A class decorator that simulates a simple form of immutability for the
293 decorated class.
295 A class decorated as `immutable` may only set each of its attributes once;
296 any attempts to set an already-set attribute will raise `AttributeError`.
298 Notes
299 -----
300 Subclasses of classes marked with ``@immutable`` are also immutable.
302 Because this behavior interferes with the default implementation for the
303 ``pickle`` modules, `immutable` provides implementations of
304 ``__getstate__`` and ``__setstate__`` that override this behavior.
305 Immutable classes can then implement pickle via ``__reduce__`` or
306 ``__getnewargs__``.
308 Following the example of Python's built-in immutable types, such as `str`
309 and `tuple`, the `immutable` decorator provides a ``__copy__``
310 implementation that just returns ``self``, because there is no reason to
311 actually copy an object if none of its shared owners can modify it.
313 Similarly, objects that are recursively (i.e. are themselves immutable and
314 have only recursively immutable attributes) should also reimplement
315 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as
316 it has no way of checking for recursive immutability.
317 """
318 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807
319 if hasattr(self, name):
320 raise AttributeError(f"{cls.__name__} instances are immutable.")
321 object.__setattr__(self, name, value)
322 # mypy says the variable here has signature (str, Any) i.e. no "self";
323 # I think it's just confused by descriptor stuff.
324 cls.__setattr__ = __setattr__ # type: ignore
326 def __getstate__(self: _T) -> dict: # noqa: N807
327 # Disable default state-setting when unpickled.
328 return {}
329 cls.__getstate__ = __getstate__
331 def __setstate__(self: _T, state: Any) -> None: # noqa: N807
332 # Disable default state-setting when copied.
333 # Sadly what works for pickle doesn't work for copy.
334 assert not state
335 cls.__setstate__ = __setstate__
337 def __copy__(self: _T) -> _T: # noqa: N807
338 return self
339 cls.__copy__ = __copy__
340 return cls
343_S = TypeVar("_S")
344_R = TypeVar("_R")
347def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
348 """A decorator that caches the result of a method that takes only ``self``
349 as an argument, returning the cached result on subsequent calls.
351 Notes
352 -----
353 This is intended primarily as a stopgap for Python 3.8's more sophisticated
354 ``functools.cached_property``, but it is also explicitly compatible with
355 the `immutable` decorator, which may not be true of ``cached_property``.
357 `cached_getter` guarantees that the cached value will be stored in
358 an attribute named ``_cached_{name-of-decorated-function}``. Classes that
359 use `cached_getter` are responsible for guaranteeing that this name is not
360 otherwise used, and is included if ``__slots__`` is defined.
361 """
362 attribute = f"_cached_{func.__name__}"
364 @functools.wraps(func)
365 def inner(self: _S) -> _R:
366 if not hasattr(self, attribute):
367 object.__setattr__(self, attribute, func(self))
368 return getattr(self, attribute)
370 return inner
373def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
374 """Get the files from a list of values. If a value is a file it is added to
375 the list of returned files. If a value is a directory, all the files in
376 the directory (recursively) that match the regex will be returned.
378 Parameters
379 ----------
380 values : iterable [`str`]
381 The files to return and directories in which to look for files to
382 return.
383 regex : `str`
384 The regex to use when searching for files within directories. Optional,
385 by default returns all the found files.
387 Returns
388 -------
389 resources: `list` [`str`]
390 The passed-in files and files found in passed-in directories.
391 """
392 fileRegex = None if regex is None else re.compile(regex)
393 resources = []
395 # Find all the files of interest
396 for location in values:
397 if os.path.isdir(location):
398 for root, dirs, files in os.walk(location):
399 for name in files:
400 path = os.path.join(root, name)
401 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
402 resources.append(path)
403 else:
404 resources.append(location)
405 return resources
408def globToRegex(expressions: List[str]) -> Union[List[Pattern[str]], EllipsisType]:
409 """Translate glob-style search terms to regex.
411 If a stand-alone '*' is found in ``expressions``, or expressions is empty,
412 then the special value ``...`` will be returned, indicating that any string
413 will match.
415 Parameters
416 ----------
417 expressions : `list` [`str`]
418 A list of glob-style pattern strings to convert.
420 Returns
421 -------
422 expressions : `list` [`str`] or ``...``
423 A list of regex Patterns
424 """
425 if not expressions or "*" in expressions:
426 return Ellipsis
427 return [re.compile(fnmatch.translate(e)) for e in expressions]