Coverage for python/lsst/daf/butler/core/utils.py : 37%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "isplit",
30 "iterable",
31 "safeMakeDir",
32 "Singleton",
33 "stripIfNotNone",
34 "transactional",
35)
37import errno
38import os
39import builtins
40import fnmatch
41import functools
42import logging
43import re
44from typing import (
45 Any,
46 Callable,
47 Dict,
48 Iterable,
49 Iterator,
50 List,
51 Mapping,
52 Optional,
53 Pattern,
54 Type,
55 TypeVar,
56 TYPE_CHECKING,
57 Union,
58)
60from lsst.utils import doImport
62if TYPE_CHECKING: 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true
63 from ..registry.wildcards import Ellipsis, EllipsisType
66_LOG = logging.getLogger(__name__)
69def safeMakeDir(directory: str) -> None:
70 """Make a directory in a manner avoiding race conditions."""
71 if directory != "" and not os.path.exists(directory):
72 try:
73 os.makedirs(directory)
74 except OSError as e:
75 # Don't fail if directory exists due to race
76 if e.errno != errno.EEXIST:
77 raise e
80def iterable(a: Any) -> Iterable[Any]:
81 """Make input iterable.
83 There are three cases, when the input is:
85 - iterable, but not a `str` or Mapping -> iterate over elements
86 (e.g. ``[i for i in a]``)
87 - a `str` -> return single element iterable (e.g. ``[a]``)
88 - a Mapping -> return single element iterable
89 - not iterable -> return single element iterable (e.g. ``[a]``).
91 Parameters
92 ----------
93 a : iterable or `str` or not iterable
94 Argument to be converted to an iterable.
96 Returns
97 -------
98 i : `generator`
99 Iterable version of the input value.
100 """
101 if isinstance(a, str):
102 yield a
103 return
104 if isinstance(a, Mapping):
105 yield a
106 return
107 try:
108 yield from a
109 except Exception:
110 yield a
113def allSlots(self: Any) -> Iterator[str]:
114 """
115 Return combined ``__slots__`` for all classes in objects mro.
117 Parameters
118 ----------
119 self : `object`
120 Instance to be inspected.
122 Returns
123 -------
124 slots : `itertools.chain`
125 All the slots as an iterable.
126 """
127 from itertools import chain
128 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
131def getFullTypeName(cls: Any) -> str:
132 """Return full type name of the supplied entity.
134 Parameters
135 ----------
136 cls : `type` or `object`
137 Entity from which to obtain the full name. Can be an instance
138 or a `type`.
140 Returns
141 -------
142 name : `str`
143 Full name of type.
145 Notes
146 -----
147 Builtins are returned without the ``builtins`` specifier included. This
148 allows `str` to be returned as "str" rather than "builtins.str". Any
149 parts of the path that start with a leading underscore are removed
150 on the assumption that they are an implementation detail and the
151 entity will be hoisted into the parent namespace.
152 """
153 # If we have an instance we need to convert to a type
154 if not hasattr(cls, "__qualname__"): 154 ↛ 155line 154 didn't jump to line 155, because the condition on line 154 was never true
155 cls = type(cls)
156 if hasattr(builtins, cls.__qualname__):
157 # Special case builtins such as str and dict
158 return cls.__qualname__
160 real_name = cls.__module__ + "." + cls.__qualname__
162 # Remove components with leading underscores
163 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
165 # Consistency check
166 if real_name != cleaned_name: 166 ↛ 167line 166 didn't jump to line 167, because the condition on line 166 was never true
167 try:
168 test = doImport(cleaned_name)
169 except Exception:
170 # Could not import anything so return the real name
171 return real_name
173 # The thing we imported should match the class we started with
174 # despite the clean up. If it does not we return the real name
175 if test is not cls:
176 return real_name
178 return cleaned_name
181def getClassOf(typeOrName: Union[Type, str]) -> Type:
182 """Given the type name or a type, return the python type.
184 If a type name is given, an attempt will be made to import the type.
186 Parameters
187 ----------
188 typeOrName : `str` or Python class
189 A string describing the Python class to load or a Python type.
191 Returns
192 -------
193 type_ : `type`
194 Directly returns the Python type if a type was provided, else
195 tries to import the given string and returns the resulting type.
197 Notes
198 -----
199 This is a thin wrapper around `~lsst.utils.doImport`.
200 """
201 if isinstance(typeOrName, str):
202 cls = doImport(typeOrName)
203 else:
204 cls = typeOrName
205 return cls
208def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
209 """Given the type name or a type, instantiate an object of that type.
211 If a type name is given, an attempt will be made to import the type.
213 Parameters
214 ----------
215 typeOrName : `str` or Python class
216 A string describing the Python class to load or a Python type.
217 args : `tuple`
218 Positional arguments to use pass to the object constructor.
219 **kwargs
220 Keyword arguments to pass to object constructor.
222 Returns
223 -------
224 instance : `object`
225 Instance of the requested type, instantiated with the provided
226 parameters.
227 """
228 cls = getClassOf(typeOrName)
229 return cls(*args, **kwargs)
232class Singleton(type):
233 """Metaclass to convert a class to a Singleton.
235 If this metaclass is used the constructor for the singleton class must
236 take no arguments. This is because a singleton class will only accept
237 the arguments the first time an instance is instantiated.
238 Therefore since you do not know if the constructor has been called yet it
239 is safer to always call it with no arguments and then call a method to
240 adjust state of the singleton.
241 """
243 _instances: Dict[Type, Any] = {}
245 # Signature is intentionally not substitutable for type.__call__ (no *args,
246 # **kwargs) to require classes that use this metaclass to have no
247 # constructor arguments.
248 def __call__(cls) -> Any: # type: ignore
249 if cls not in cls._instances:
250 cls._instances[cls] = super(Singleton, cls).__call__()
251 return cls._instances[cls]
254F = TypeVar("F", bound=Callable)
257def transactional(func: F) -> F:
258 """Decorate a method and makes it transactional.
260 This depends on the class also defining a `transaction` method
261 that takes no arguments and acts as a context manager.
262 """
263 @functools.wraps(func)
264 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
265 with self.transaction():
266 return func(self, *args, **kwargs)
267 return inner # type: ignore
270def stripIfNotNone(s: Optional[str]) -> Optional[str]:
271 """Strip leading and trailing whitespace if the given object is not None.
273 Parameters
274 ----------
275 s : `str`, optional
276 Input string.
278 Returns
279 -------
280 r : `str` or `None`
281 A string with leading and trailing whitespace stripped if `s` is not
282 `None`, or `None` if `s` is `None`.
283 """
284 if s is not None:
285 s = s.strip()
286 return s
289_T = TypeVar("_T", bound="Type")
292def immutable(cls: _T) -> _T:
293 """Decorate a class to simulates a simple form of immutability.
295 A class decorated as `immutable` may only set each of its attributes once;
296 any attempts to set an already-set attribute will raise `AttributeError`.
298 Notes
299 -----
300 Subclasses of classes marked with ``@immutable`` are also immutable.
302 Because this behavior interferes with the default implementation for the
303 ``pickle`` modules, `immutable` provides implementations of
304 ``__getstate__`` and ``__setstate__`` that override this behavior.
305 Immutable classes can then implement pickle via ``__reduce__`` or
306 ``__getnewargs__``.
308 Following the example of Python's built-in immutable types, such as `str`
309 and `tuple`, the `immutable` decorator provides a ``__copy__``
310 implementation that just returns ``self``, because there is no reason to
311 actually copy an object if none of its shared owners can modify it.
313 Similarly, objects that are recursively (i.e. are themselves immutable and
314 have only recursively immutable attributes) should also reimplement
315 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as
316 it has no way of checking for recursive immutability.
317 """
318 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807
319 if hasattr(self, name):
320 raise AttributeError(f"{cls.__name__} instances are immutable.")
321 object.__setattr__(self, name, value)
322 # mypy says the variable here has signature (str, Any) i.e. no "self";
323 # I think it's just confused by descriptor stuff.
324 cls.__setattr__ = __setattr__ # type: ignore
326 def __getstate__(self: _T) -> dict: # noqa: N807
327 # Disable default state-setting when unpickled.
328 return {}
329 cls.__getstate__ = __getstate__
331 def __setstate__(self: _T, state: Any) -> None: # noqa: N807
332 # Disable default state-setting when copied.
333 # Sadly what works for pickle doesn't work for copy.
334 assert not state
335 cls.__setstate__ = __setstate__
337 def __copy__(self: _T) -> _T: # noqa: N807
338 return self
339 cls.__copy__ = __copy__
340 return cls
343_S = TypeVar("_S")
344_R = TypeVar("_R")
347def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
348 """Decorate a method to caches the result.
350 Only works on methods that take only ``self``
351 as an argument, and returns the cached result on subsequent calls.
353 Notes
354 -----
355 This is intended primarily as a stopgap for Python 3.8's more sophisticated
356 ``functools.cached_property``, but it is also explicitly compatible with
357 the `immutable` decorator, which may not be true of ``cached_property``.
359 `cached_getter` guarantees that the cached value will be stored in
360 an attribute named ``_cached_{name-of-decorated-function}``. Classes that
361 use `cached_getter` are responsible for guaranteeing that this name is not
362 otherwise used, and is included if ``__slots__`` is defined.
363 """
364 attribute = f"_cached_{func.__name__}"
366 @functools.wraps(func)
367 def inner(self: _S) -> _R:
368 if not hasattr(self, attribute):
369 object.__setattr__(self, attribute, func(self))
370 return getattr(self, attribute)
372 return inner
375def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
376 """Scan the supplied directories and return all matching files.
378 Get the files from a list of values. If a value is a file it is added to
379 the list of returned files. If a value is a directory, all the files in
380 the directory (recursively) that match the regex will be returned.
382 Parameters
383 ----------
384 values : iterable [`str`]
385 The files to return and directories in which to look for files to
386 return.
387 regex : `str`
388 The regex to use when searching for files within directories. Optional,
389 by default returns all the found files.
391 Returns
392 -------
393 resources: `list` [`str`]
394 The passed-in files and files found in passed-in directories.
395 """
396 fileRegex = None if regex is None else re.compile(regex)
397 resources = []
399 # Find all the files of interest
400 for location in values:
401 if os.path.isdir(location):
402 for root, dirs, files in os.walk(location):
403 for name in files:
404 path = os.path.join(root, name)
405 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
406 resources.append(path)
407 else:
408 resources.append(location)
409 return resources
412def globToRegex(expressions: Union[str, EllipsisType, None,
413 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]:
414 """Translate glob-style search terms to regex.
416 If a stand-alone '``*``' is found in ``expressions``, or expressions is
417 empty or `None`, then the special value ``...`` will be returned,
418 indicating that any string will match.
420 Parameters
421 ----------
422 expressions : `str` or `list` [`str`]
423 A list of glob-style pattern strings to convert.
425 Returns
426 -------
427 expressions : `list` [`str` or `re.Pattern`] or ``...``
428 A list of regex Patterns or simple strings. Returns ``...`` if
429 the provided expressions would match everything.
430 """
431 if expressions is Ellipsis or expressions is None:
432 return Ellipsis
433 expressions = list(iterable(expressions))
434 if not expressions or "*" in expressions:
435 return Ellipsis
437 nomagic = re.compile(r"^[\w/\.\-]+$", re.ASCII)
439 # Try not to convert simple string to a regex.
440 results: List[Union[str, Pattern]] = []
441 for e in expressions:
442 res: Union[str, Pattern]
443 if nomagic.match(e):
444 res = e
445 else:
446 res = re.compile(fnmatch.translate(e))
447 results.append(res)
448 return results
451T = TypeVar('T', str, bytes)
454def isplit(string: T, sep: T) -> Iterator[T]:
455 """Split a string or bytes by separator returning a generator.
457 Parameters
458 ----------
459 string : `str` or `bytes`
460 The string to split into substrings.
461 sep : `str` or `bytes`
462 The separator to use to split the string. Must be the same
463 type as ``string``. Must always be given.
465 Yields
466 ------
467 subset : `str` or `bytes`
468 The next subset extracted from the input until the next separator.
469 """
470 begin = 0
471 while True:
472 end = string.find(sep, begin)
473 if end == -1:
474 yield string[begin:]
475 return
476 yield string[begin:end]
477 begin = end + 1