Coverage for python/lsst/daf/butler/core/utils.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "iterable",
30 "safeMakeDir",
31 "Singleton",
32 "stripIfNotNone",
33 "transactional",
34)
36import errno
37import os
38import builtins
39import fnmatch
40import functools
41import logging
42import re
43from typing import (
44 Any,
45 Callable,
46 Dict,
47 Iterable,
48 Iterator,
49 List,
50 Mapping,
51 Optional,
52 Pattern,
53 Type,
54 TypeVar,
55 TYPE_CHECKING,
56 Union,
57)
59from lsst.utils import doImport
61if TYPE_CHECKING: 61 ↛ 62line 61 didn't jump to line 62, because the condition on line 61 was never true
62 from ..registry.wildcards import Ellipsis, EllipsisType
65_LOG = logging.getLogger(__name__)
68def safeMakeDir(directory: str) -> None:
69 """Make a directory in a manner avoiding race conditions."""
70 if directory != "" and not os.path.exists(directory):
71 try:
72 os.makedirs(directory)
73 except OSError as e:
74 # Don't fail if directory exists due to race
75 if e.errno != errno.EEXIST:
76 raise e
79def iterable(a: Any) -> Iterable[Any]:
80 """Make input iterable.
82 There are three cases, when the input is:
84 - iterable, but not a `str` or Mapping -> iterate over elements
85 (e.g. ``[i for i in a]``)
86 - a `str` -> return single element iterable (e.g. ``[a]``)
87 - a Mapping -> return single element iterable
88 - not iterable -> return single element iterable (e.g. ``[a]``).
90 Parameters
91 ----------
92 a : iterable or `str` or not iterable
93 Argument to be converted to an iterable.
95 Returns
96 -------
97 i : `generator`
98 Iterable version of the input value.
99 """
100 if isinstance(a, str):
101 yield a
102 return
103 if isinstance(a, Mapping):
104 yield a
105 return
106 try:
107 yield from a
108 except Exception:
109 yield a
112def allSlots(self: Any) -> Iterator[str]:
113 """
114 Return combined ``__slots__`` for all classes in objects mro.
116 Parameters
117 ----------
118 self : `object`
119 Instance to be inspected.
121 Returns
122 -------
123 slots : `itertools.chain`
124 All the slots as an iterable.
125 """
126 from itertools import chain
127 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
130def getFullTypeName(cls: Any) -> str:
131 """Return full type name of the supplied entity.
133 Parameters
134 ----------
135 cls : `type` or `object`
136 Entity from which to obtain the full name. Can be an instance
137 or a `type`.
139 Returns
140 -------
141 name : `str`
142 Full name of type.
144 Notes
145 -----
146 Builtins are returned without the ``builtins`` specifier included. This
147 allows `str` to be returned as "str" rather than "builtins.str". Any
148 parts of the path that start with a leading underscore are removed
149 on the assumption that they are an implementation detail and the
150 entity will be hoisted into the parent namespace.
151 """
152 # If we have an instance we need to convert to a type
153 if not hasattr(cls, "__qualname__"): 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 cls = type(cls)
155 if hasattr(builtins, cls.__qualname__):
156 # Special case builtins such as str and dict
157 return cls.__qualname__
159 real_name = cls.__module__ + "." + cls.__qualname__
161 # Remove components with leading underscores
162 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
164 # Consistency check
165 if real_name != cleaned_name: 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true
166 try:
167 test = doImport(cleaned_name)
168 except Exception:
169 # Could not import anything so return the real name
170 return real_name
172 # The thing we imported should match the class we started with
173 # despite the clean up. If it does not we return the real name
174 if test is not cls:
175 return real_name
177 return cleaned_name
180def getClassOf(typeOrName: Union[Type, str]) -> Type:
181 """Given the type name or a type, return the python type.
183 If a type name is given, an attempt will be made to import the type.
185 Parameters
186 ----------
187 typeOrName : `str` or Python class
188 A string describing the Python class to load or a Python type.
190 Returns
191 -------
192 type_ : `type`
193 Directly returns the Python type if a type was provided, else
194 tries to import the given string and returns the resulting type.
196 Notes
197 -----
198 This is a thin wrapper around `~lsst.utils.doImport`.
199 """
200 if isinstance(typeOrName, str):
201 cls = doImport(typeOrName)
202 else:
203 cls = typeOrName
204 return cls
207def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
208 """Given the type name or a type, instantiate an object of that type.
210 If a type name is given, an attempt will be made to import the type.
212 Parameters
213 ----------
214 typeOrName : `str` or Python class
215 A string describing the Python class to load or a Python type.
216 args : `tuple`
217 Positional arguments to use pass to the object constructor.
218 kwargs : `dict`
219 Keyword arguments to pass to object constructor.
221 Returns
222 -------
223 instance : `object`
224 Instance of the requested type, instantiated with the provided
225 parameters.
226 """
227 cls = getClassOf(typeOrName)
228 return cls(*args, **kwargs)
231class Singleton(type):
232 """Metaclass to convert a class to a Singleton.
234 If this metaclass is used the constructor for the singleton class must
235 take no arguments. This is because a singleton class will only accept
236 the arguments the first time an instance is instantiated.
237 Therefore since you do not know if the constructor has been called yet it
238 is safer to always call it with no arguments and then call a method to
239 adjust state of the singleton.
240 """
242 _instances: Dict[Type, Any] = {}
244 # Signature is intentionally not substitutable for type.__call__ (no *args,
245 # **kwargs) to require classes that use this metaclass to have no
246 # constructor arguments.
247 def __call__(cls) -> Any: # type: ignore
248 if cls not in cls._instances:
249 cls._instances[cls] = super(Singleton, cls).__call__()
250 return cls._instances[cls]
253F = TypeVar("F", bound=Callable)
256def transactional(func: F) -> F:
257 """Decorate a method and makes it transactional.
259 This depends on the class also defining a `transaction` method
260 that takes no arguments and acts as a context manager.
261 """
262 @functools.wraps(func)
263 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
264 with self.transaction():
265 return func(self, *args, **kwargs)
266 return inner # type: ignore
269def stripIfNotNone(s: Optional[str]) -> Optional[str]:
270 """Strip leading and trailing whitespace if the given object is not None.
272 Parameters
273 ----------
274 s : `str`, optional
275 Input string.
277 Returns
278 -------
279 r : `str` or `None`
280 A string with leading and trailing whitespace stripped if `s` is not
281 `None`, or `None` if `s` is `None`.
282 """
283 if s is not None:
284 s = s.strip()
285 return s
288_T = TypeVar("_T", bound="Type")
291def immutable(cls: _T) -> _T:
292 """Decorate a class to simulates a simple form of immutability.
294 A class decorated as `immutable` may only set each of its attributes once;
295 any attempts to set an already-set attribute will raise `AttributeError`.
297 Notes
298 -----
299 Subclasses of classes marked with ``@immutable`` are also immutable.
301 Because this behavior interferes with the default implementation for the
302 ``pickle`` modules, `immutable` provides implementations of
303 ``__getstate__`` and ``__setstate__`` that override this behavior.
304 Immutable classes can then implement pickle via ``__reduce__`` or
305 ``__getnewargs__``.
307 Following the example of Python's built-in immutable types, such as `str`
308 and `tuple`, the `immutable` decorator provides a ``__copy__``
309 implementation that just returns ``self``, because there is no reason to
310 actually copy an object if none of its shared owners can modify it.
312 Similarly, objects that are recursively (i.e. are themselves immutable and
313 have only recursively immutable attributes) should also reimplement
314 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as
315 it has no way of checking for recursive immutability.
316 """
317 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807
318 if hasattr(self, name):
319 raise AttributeError(f"{cls.__name__} instances are immutable.")
320 object.__setattr__(self, name, value)
321 # mypy says the variable here has signature (str, Any) i.e. no "self";
322 # I think it's just confused by descriptor stuff.
323 cls.__setattr__ = __setattr__ # type: ignore
325 def __getstate__(self: _T) -> dict: # noqa: N807
326 # Disable default state-setting when unpickled.
327 return {}
328 cls.__getstate__ = __getstate__
330 def __setstate__(self: _T, state: Any) -> None: # noqa: N807
331 # Disable default state-setting when copied.
332 # Sadly what works for pickle doesn't work for copy.
333 assert not state
334 cls.__setstate__ = __setstate__
336 def __copy__(self: _T) -> _T: # noqa: N807
337 return self
338 cls.__copy__ = __copy__
339 return cls
342_S = TypeVar("_S")
343_R = TypeVar("_R")
346def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
347 """Decorate a method to caches the result.
349 Only works on methods that take only ``self``
350 as an argument, and returns the cached result on subsequent calls.
352 Notes
353 -----
354 This is intended primarily as a stopgap for Python 3.8's more sophisticated
355 ``functools.cached_property``, but it is also explicitly compatible with
356 the `immutable` decorator, which may not be true of ``cached_property``.
358 `cached_getter` guarantees that the cached value will be stored in
359 an attribute named ``_cached_{name-of-decorated-function}``. Classes that
360 use `cached_getter` are responsible for guaranteeing that this name is not
361 otherwise used, and is included if ``__slots__`` is defined.
362 """
363 attribute = f"_cached_{func.__name__}"
365 @functools.wraps(func)
366 def inner(self: _S) -> _R:
367 if not hasattr(self, attribute):
368 object.__setattr__(self, attribute, func(self))
369 return getattr(self, attribute)
371 return inner
374def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
375 """Scan the supplied directories and return all matching files.
377 Get the files from a list of values. If a value is a file it is added to
378 the list of returned files. If a value is a directory, all the files in
379 the directory (recursively) that match the regex will be returned.
381 Parameters
382 ----------
383 values : iterable [`str`]
384 The files to return and directories in which to look for files to
385 return.
386 regex : `str`
387 The regex to use when searching for files within directories. Optional,
388 by default returns all the found files.
390 Returns
391 -------
392 resources: `list` [`str`]
393 The passed-in files and files found in passed-in directories.
394 """
395 fileRegex = None if regex is None else re.compile(regex)
396 resources = []
398 # Find all the files of interest
399 for location in values:
400 if os.path.isdir(location):
401 for root, dirs, files in os.walk(location):
402 for name in files:
403 path = os.path.join(root, name)
404 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
405 resources.append(path)
406 else:
407 resources.append(location)
408 return resources
411def globToRegex(expressions: Union[str, EllipsisType, None,
412 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]:
413 """Translate glob-style search terms to regex.
415 If a stand-alone '``*``' is found in ``expressions``, or expressions is
416 empty or `None`, then the special value ``...`` will be returned,
417 indicating that any string will match.
419 Parameters
420 ----------
421 expressions : `str` or `list` [`str`]
422 A list of glob-style pattern strings to convert.
424 Returns
425 -------
426 expressions : `list` [`str` or `re.Pattern`] or ``...``
427 A list of regex Patterns or simple strings. Returns ``...`` if
428 the provided expressions would match everything.
429 """
430 if expressions is Ellipsis or expressions is None:
431 return Ellipsis
432 expressions = list(iterable(expressions))
433 if not expressions or "*" in expressions:
434 return Ellipsis
436 nomagic = re.compile(r"^[\w/\.\-]+$", re.ASCII)
438 # Try not to convert simple string to a regex.
439 results: List[Union[str, Pattern]] = []
440 for e in expressions:
441 res: Union[str, Pattern]
442 if nomagic.match(e):
443 res = e
444 else:
445 res = re.compile(fnmatch.translate(e))
446 results.append(res)
447 return results