Coverage for python/lsst/daf/butler/core/utils.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "isplit",
30 "iterable",
31 "safeMakeDir",
32 "Singleton",
33 "stripIfNotNone",
34 "time_this",
35 "transactional",
36)
38import errno
39import os
40import builtins
41import fnmatch
42import functools
43import logging
44import time
45import re
46from contextlib import contextmanager
47from typing import (
48 Any,
49 Callable,
50 Dict,
51 Iterable,
52 Iterator,
53 List,
54 Mapping,
55 Optional,
56 Pattern,
57 Type,
58 TypeVar,
59 TYPE_CHECKING,
60 Union,
61)
63from lsst.utils import doImport
65if TYPE_CHECKING: 65 ↛ 66line 65 didn't jump to line 66, because the condition on line 65 was never true
66 from ..registry.wildcards import Ellipsis, EllipsisType
69_LOG = logging.getLogger(__name__)
72def safeMakeDir(directory: str) -> None:
73 """Make a directory in a manner avoiding race conditions."""
74 if directory != "" and not os.path.exists(directory):
75 try:
76 os.makedirs(directory)
77 except OSError as e:
78 # Don't fail if directory exists due to race
79 if e.errno != errno.EEXIST:
80 raise e
83def iterable(a: Any) -> Iterable[Any]:
84 """Make input iterable.
86 There are three cases, when the input is:
88 - iterable, but not a `str` or Mapping -> iterate over elements
89 (e.g. ``[i for i in a]``)
90 - a `str` -> return single element iterable (e.g. ``[a]``)
91 - a Mapping -> return single element iterable
92 - not iterable -> return single element iterable (e.g. ``[a]``).
94 Parameters
95 ----------
96 a : iterable or `str` or not iterable
97 Argument to be converted to an iterable.
99 Returns
100 -------
101 i : `generator`
102 Iterable version of the input value.
103 """
104 if isinstance(a, str):
105 yield a
106 return
107 if isinstance(a, Mapping):
108 yield a
109 return
110 try:
111 yield from a
112 except Exception:
113 yield a
116def allSlots(self: Any) -> Iterator[str]:
117 """
118 Return combined ``__slots__`` for all classes in objects mro.
120 Parameters
121 ----------
122 self : `object`
123 Instance to be inspected.
125 Returns
126 -------
127 slots : `itertools.chain`
128 All the slots as an iterable.
129 """
130 from itertools import chain
131 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
134def getFullTypeName(cls: Any) -> str:
135 """Return full type name of the supplied entity.
137 Parameters
138 ----------
139 cls : `type` or `object`
140 Entity from which to obtain the full name. Can be an instance
141 or a `type`.
143 Returns
144 -------
145 name : `str`
146 Full name of type.
148 Notes
149 -----
150 Builtins are returned without the ``builtins`` specifier included. This
151 allows `str` to be returned as "str" rather than "builtins.str". Any
152 parts of the path that start with a leading underscore are removed
153 on the assumption that they are an implementation detail and the
154 entity will be hoisted into the parent namespace.
155 """
156 # If we have an instance we need to convert to a type
157 if not hasattr(cls, "__qualname__"): 157 ↛ 158line 157 didn't jump to line 158, because the condition on line 157 was never true
158 cls = type(cls)
159 if hasattr(builtins, cls.__qualname__):
160 # Special case builtins such as str and dict
161 return cls.__qualname__
163 real_name = cls.__module__ + "." + cls.__qualname__
165 # Remove components with leading underscores
166 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
168 # Consistency check
169 if real_name != cleaned_name: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true
170 try:
171 test = doImport(cleaned_name)
172 except Exception:
173 # Could not import anything so return the real name
174 return real_name
176 # The thing we imported should match the class we started with
177 # despite the clean up. If it does not we return the real name
178 if test is not cls:
179 return real_name
181 return cleaned_name
184def getClassOf(typeOrName: Union[Type, str]) -> Type:
185 """Given the type name or a type, return the python type.
187 If a type name is given, an attempt will be made to import the type.
189 Parameters
190 ----------
191 typeOrName : `str` or Python class
192 A string describing the Python class to load or a Python type.
194 Returns
195 -------
196 type_ : `type`
197 Directly returns the Python type if a type was provided, else
198 tries to import the given string and returns the resulting type.
200 Notes
201 -----
202 This is a thin wrapper around `~lsst.utils.doImport`.
203 """
204 if isinstance(typeOrName, str):
205 cls = doImport(typeOrName)
206 else:
207 cls = typeOrName
208 return cls
211def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
212 """Given the type name or a type, instantiate an object of that type.
214 If a type name is given, an attempt will be made to import the type.
216 Parameters
217 ----------
218 typeOrName : `str` or Python class
219 A string describing the Python class to load or a Python type.
220 args : `tuple`
221 Positional arguments to use pass to the object constructor.
222 **kwargs
223 Keyword arguments to pass to object constructor.
225 Returns
226 -------
227 instance : `object`
228 Instance of the requested type, instantiated with the provided
229 parameters.
230 """
231 cls = getClassOf(typeOrName)
232 return cls(*args, **kwargs)
235class Singleton(type):
236 """Metaclass to convert a class to a Singleton.
238 If this metaclass is used the constructor for the singleton class must
239 take no arguments. This is because a singleton class will only accept
240 the arguments the first time an instance is instantiated.
241 Therefore since you do not know if the constructor has been called yet it
242 is safer to always call it with no arguments and then call a method to
243 adjust state of the singleton.
244 """
246 _instances: Dict[Type, Any] = {}
248 # Signature is intentionally not substitutable for type.__call__ (no *args,
249 # **kwargs) to require classes that use this metaclass to have no
250 # constructor arguments.
251 def __call__(cls) -> Any: # type: ignore
252 if cls not in cls._instances:
253 cls._instances[cls] = super(Singleton, cls).__call__()
254 return cls._instances[cls]
257F = TypeVar("F", bound=Callable)
260def transactional(func: F) -> F:
261 """Decorate a method and makes it transactional.
263 This depends on the class also defining a `transaction` method
264 that takes no arguments and acts as a context manager.
265 """
266 @functools.wraps(func)
267 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
268 with self.transaction():
269 return func(self, *args, **kwargs)
270 return inner # type: ignore
273def stripIfNotNone(s: Optional[str]) -> Optional[str]:
274 """Strip leading and trailing whitespace if the given object is not None.
276 Parameters
277 ----------
278 s : `str`, optional
279 Input string.
281 Returns
282 -------
283 r : `str` or `None`
284 A string with leading and trailing whitespace stripped if `s` is not
285 `None`, or `None` if `s` is `None`.
286 """
287 if s is not None:
288 s = s.strip()
289 return s
292_T = TypeVar("_T", bound="Type")
295def immutable(cls: _T) -> _T:
296 """Decorate a class to simulates a simple form of immutability.
298 A class decorated as `immutable` may only set each of its attributes once;
299 any attempts to set an already-set attribute will raise `AttributeError`.
301 Notes
302 -----
303 Subclasses of classes marked with ``@immutable`` are also immutable.
305 Because this behavior interferes with the default implementation for the
306 ``pickle`` modules, `immutable` provides implementations of
307 ``__getstate__`` and ``__setstate__`` that override this behavior.
308 Immutable classes can then implement pickle via ``__reduce__`` or
309 ``__getnewargs__``.
311 Following the example of Python's built-in immutable types, such as `str`
312 and `tuple`, the `immutable` decorator provides a ``__copy__``
313 implementation that just returns ``self``, because there is no reason to
314 actually copy an object if none of its shared owners can modify it.
316 Similarly, objects that are recursively (i.e. are themselves immutable and
317 have only recursively immutable attributes) should also reimplement
318 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as
319 it has no way of checking for recursive immutability.
320 """
321 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807
322 if hasattr(self, name):
323 raise AttributeError(f"{cls.__name__} instances are immutable.")
324 object.__setattr__(self, name, value)
325 # mypy says the variable here has signature (str, Any) i.e. no "self";
326 # I think it's just confused by descriptor stuff.
327 cls.__setattr__ = __setattr__ # type: ignore
329 def __getstate__(self: _T) -> dict: # noqa: N807
330 # Disable default state-setting when unpickled.
331 return {}
332 cls.__getstate__ = __getstate__
334 def __setstate__(self: _T, state: Any) -> None: # noqa: N807
335 # Disable default state-setting when copied.
336 # Sadly what works for pickle doesn't work for copy.
337 assert not state
338 cls.__setstate__ = __setstate__
340 def __copy__(self: _T) -> _T: # noqa: N807
341 return self
342 cls.__copy__ = __copy__
343 return cls
346_S = TypeVar("_S")
347_R = TypeVar("_R")
350def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
351 """Decorate a method to caches the result.
353 Only works on methods that take only ``self``
354 as an argument, and returns the cached result on subsequent calls.
356 Notes
357 -----
358 This is intended primarily as a stopgap for Python 3.8's more sophisticated
359 ``functools.cached_property``, but it is also explicitly compatible with
360 the `immutable` decorator, which may not be true of ``cached_property``.
362 `cached_getter` guarantees that the cached value will be stored in
363 an attribute named ``_cached_{name-of-decorated-function}``. Classes that
364 use `cached_getter` are responsible for guaranteeing that this name is not
365 otherwise used, and is included if ``__slots__`` is defined.
366 """
367 attribute = f"_cached_{func.__name__}"
369 @functools.wraps(func)
370 def inner(self: _S) -> _R:
371 if not hasattr(self, attribute):
372 object.__setattr__(self, attribute, func(self))
373 return getattr(self, attribute)
375 return inner
378def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
379 """Scan the supplied directories and return all matching files.
381 Get the files from a list of values. If a value is a file it is added to
382 the list of returned files. If a value is a directory, all the files in
383 the directory (recursively) that match the regex will be returned.
385 Parameters
386 ----------
387 values : iterable [`str`]
388 The files to return and directories in which to look for files to
389 return.
390 regex : `str`
391 The regex to use when searching for files within directories. Optional,
392 by default returns all the found files.
394 Returns
395 -------
396 resources: `list` [`str`]
397 The passed-in files and files found in passed-in directories.
398 """
399 fileRegex = None if regex is None else re.compile(regex)
400 resources = []
402 # Find all the files of interest
403 for location in values:
404 if os.path.isdir(location):
405 for root, dirs, files in os.walk(location):
406 for name in files:
407 path = os.path.join(root, name)
408 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
409 resources.append(path)
410 else:
411 resources.append(location)
412 return resources
415def globToRegex(expressions: Union[str, EllipsisType, None,
416 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]:
417 """Translate glob-style search terms to regex.
419 If a stand-alone '``*``' is found in ``expressions``, or expressions is
420 empty or `None`, then the special value ``...`` will be returned,
421 indicating that any string will match.
423 Parameters
424 ----------
425 expressions : `str` or `list` [`str`]
426 A list of glob-style pattern strings to convert.
428 Returns
429 -------
430 expressions : `list` [`str` or `re.Pattern`] or ``...``
431 A list of regex Patterns or simple strings. Returns ``...`` if
432 the provided expressions would match everything.
433 """
434 if expressions is Ellipsis or expressions is None:
435 return Ellipsis
436 expressions = list(iterable(expressions))
437 if not expressions or "*" in expressions:
438 return Ellipsis
440 nomagic = re.compile(r"^[\w/\.\-]+$", re.ASCII)
442 # Try not to convert simple string to a regex.
443 results: List[Union[str, Pattern]] = []
444 for e in expressions:
445 res: Union[str, Pattern]
446 if nomagic.match(e):
447 res = e
448 else:
449 res = re.compile(fnmatch.translate(e))
450 results.append(res)
451 return results
454T = TypeVar('T', str, bytes)
457def isplit(string: T, sep: T) -> Iterator[T]:
458 """Split a string or bytes by separator returning a generator.
460 Parameters
461 ----------
462 string : `str` or `bytes`
463 The string to split into substrings.
464 sep : `str` or `bytes`
465 The separator to use to split the string. Must be the same
466 type as ``string``. Must always be given.
468 Yields
469 ------
470 subset : `str` or `bytes`
471 The next subset extracted from the input until the next separator.
472 """
473 begin = 0
474 while True:
475 end = string.find(sep, begin)
476 if end == -1:
477 yield string[begin:]
478 return
479 yield string[begin:end]
480 begin = end + 1
483@contextmanager
484def time_this(log: Optional[logging.Logger] = None, msg: Optional[str] = None,
485 level: int = logging.DEBUG, prefix: Optional[str] = "timer",
486 args: Iterable[Any] = ()) -> Iterator[None]:
487 """Time the enclosed block and issue a log message.
489 Parameters
490 ----------
491 log : `logging.Logger`, optional
492 Logger to use to report the timer message. The root logger will
493 be used if none is given.
494 msg : `str`, optional
495 Context to include in log message.
496 level : `int`, optional
497 Python logging level to use to issue the log message. If the
498 code block raises an exception the log message will automatically
499 switch to level ERROR.
500 prefix : `str`, optional
501 Prefix to use to prepend to the supplied logger to
502 create a new logger to use instead. No prefix is used if the value
503 is set to `None`. Defaults to "timer".
504 args : iterable of any
505 Additional parameters passed to the log command that should be
506 written to ``msg``.
507 """
508 if log is None:
509 log = logging.getLogger()
510 if prefix:
511 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
512 log = logging.getLogger(log_name)
514 success = False
515 start = time.time()
516 try:
517 yield
518 success = True
519 finally:
520 end = time.time()
522 # The message is pre-inserted to allow the logger to expand
523 # the additional args provided. Make that easier by converting
524 # the None message to empty string.
525 if msg is None:
526 msg = ""
528 if not success:
529 # Something went wrong so change the log level to indicate
530 # this.
531 level = logging.ERROR
533 # Specify stacklevel to ensure the message is reported from the
534 # caller (1 is this file, 2 is contextlib, 3 is user)
535 log.log(level, msg + "%sTook %.4f seconds", *args,
536 ": " if msg else "", end - start, stacklevel=3)