Coverage for python/lsst/daf/butler/core/utils.py : 39%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "iterable",
30 "safeMakeDir",
31 "Singleton",
32 "stripIfNotNone",
33 "transactional",
34)
36import errno
37import os
38import builtins
39import fnmatch
40import functools
41import logging
42import re
43from typing import (
44 Any,
45 Callable,
46 Dict,
47 Iterable,
48 Iterator,
49 List,
50 Mapping,
51 Optional,
52 Pattern,
53 Type,
54 TypeVar,
55 Union,
56)
58from lsst.utils import doImport
61_LOG = logging.getLogger(__name__)
64def safeMakeDir(directory: str) -> None:
65 """Make a directory in a manner avoiding race conditions"""
66 if directory != "" and not os.path.exists(directory):
67 try:
68 os.makedirs(directory)
69 except OSError as e:
70 # Don't fail if directory exists due to race
71 if e.errno != errno.EEXIST:
72 raise e
75def iterable(a: Any) -> Iterable[Any]:
76 """Make input iterable.
78 There are three cases, when the input is:
80 - iterable, but not a `str` or Mapping -> iterate over elements
81 (e.g. ``[i for i in a]``)
82 - a `str` -> return single element iterable (e.g. ``[a]``)
83 - a Mapping -> return single element iterable
84 - not iterable -> return single element iterable (e.g. ``[a]``).
86 Parameters
87 ----------
88 a : iterable or `str` or not iterable
89 Argument to be converted to an iterable.
91 Returns
92 -------
93 i : `generator`
94 Iterable version of the input value.
95 """
96 if isinstance(a, str):
97 yield a
98 return
99 if isinstance(a, Mapping):
100 yield a
101 return
102 try:
103 yield from a
104 except Exception:
105 yield a
108def allSlots(self: Any) -> Iterator[str]:
109 """
110 Return combined ``__slots__`` for all classes in objects mro.
112 Parameters
113 ----------
114 self : `object`
115 Instance to be inspected.
117 Returns
118 -------
119 slots : `itertools.chain`
120 All the slots as an iterable.
121 """
122 from itertools import chain
123 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
126def getFullTypeName(cls: Any) -> str:
127 """Return full type name of the supplied entity.
129 Parameters
130 ----------
131 cls : `type` or `object`
132 Entity from which to obtain the full name. Can be an instance
133 or a `type`.
135 Returns
136 -------
137 name : `str`
138 Full name of type.
140 Notes
141 -----
142 Builtins are returned without the ``builtins`` specifier included. This
143 allows `str` to be returned as "str" rather than "builtins.str". Any
144 parts of the path that start with a leading underscore are removed
145 on the assumption that they are an implementation detail and the
146 entity will be hoisted into the parent namespace.
147 """
148 # If we have an instance we need to convert to a type
149 if not hasattr(cls, "__qualname__"): 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true
150 cls = type(cls)
151 if hasattr(builtins, cls.__qualname__):
152 # Special case builtins such as str and dict
153 return cls.__qualname__
155 real_name = cls.__module__ + "." + cls.__qualname__
157 # Remove components with leading underscores
158 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
160 # Consistency check
161 if real_name != cleaned_name: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true
162 try:
163 test = doImport(cleaned_name)
164 except Exception:
165 # Could not import anything so return the real name
166 return real_name
168 # The thing we imported should match the class we started with
169 # despite the clean up. If it does not we return the real name
170 if test is not cls:
171 return real_name
173 return cleaned_name
176def getClassOf(typeOrName: Union[Type, str]) -> Type:
177 """Given the type name or a type, return the python type.
179 If a type name is given, an attempt will be made to import the type.
181 Parameters
182 ----------
183 typeOrName : `str` or Python class
184 A string describing the Python class to load or a Python type.
186 Returns
187 -------
188 type_ : `type`
189 Directly returns the Python type if a type was provided, else
190 tries to import the given string and returns the resulting type.
192 Notes
193 -----
194 This is a thin wrapper around `~lsst.utils.doImport`.
195 """
196 if isinstance(typeOrName, str):
197 cls = doImport(typeOrName)
198 else:
199 cls = typeOrName
200 return cls
203def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
204 """Given the type name or a type, instantiate an object of that type.
206 If a type name is given, an attempt will be made to import the type.
208 Parameters
209 ----------
210 typeOrName : `str` or Python class
211 A string describing the Python class to load or a Python type.
212 args : `tuple`
213 Positional arguments to use pass to the object constructor.
214 kwargs : `dict`
215 Keyword arguments to pass to object constructor.
217 Returns
218 -------
219 instance : `object`
220 Instance of the requested type, instantiated with the provided
221 parameters.
222 """
223 cls = getClassOf(typeOrName)
224 return cls(*args, **kwargs)
227class Singleton(type):
228 """Metaclass to convert a class to a Singleton.
230 If this metaclass is used the constructor for the singleton class must
231 take no arguments. This is because a singleton class will only accept
232 the arguments the first time an instance is instantiated.
233 Therefore since you do not know if the constructor has been called yet it
234 is safer to always call it with no arguments and then call a method to
235 adjust state of the singleton.
236 """
238 _instances: Dict[Type, Any] = {}
240 # Signature is intentionally not substitutable for type.__call__ (no *args,
241 # **kwargs) to require classes that use this metaclass to have no
242 # constructor arguments.
243 def __call__(cls) -> Any: # type: ignore
244 if cls not in cls._instances: 244 ↛ 246line 244 didn't jump to line 246, because the condition on line 244 was never false
245 cls._instances[cls] = super(Singleton, cls).__call__()
246 return cls._instances[cls]
249F = TypeVar("F", bound=Callable)
252def transactional(func: F) -> F:
253 """Decorator that wraps a method and makes it transactional.
255 This depends on the class also defining a `transaction` method
256 that takes no arguments and acts as a context manager.
257 """
258 @functools.wraps(func)
259 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
260 with self.transaction():
261 return func(self, *args, **kwargs)
262 return inner # type: ignore
265def stripIfNotNone(s: Optional[str]) -> Optional[str]:
266 """Strip leading and trailing whitespace if the given object is not None.
268 Parameters
269 ----------
270 s : `str`, optional
271 Input string.
273 Returns
274 -------
275 r : `str` or `None`
276 A string with leading and trailing whitespace stripped if `s` is not
277 `None`, or `None` if `s` is `None`.
278 """
279 if s is not None:
280 s = s.strip()
281 return s
284_T = TypeVar("_T", bound="Type")
287def immutable(cls: _T) -> _T:
288 """A class decorator that simulates a simple form of immutability for the
289 decorated class.
291 A class decorated as `immutable` may only set each of its attributes once;
292 any attempts to set an already-set attribute will raise `AttributeError`.
294 Notes
295 -----
296 Subclasses of classes marked with ``@immutable`` are also immutable.
298 Because this behavior interferes with the default implementation for the
299 ``pickle`` modules, `immutable` provides implementations of
300 ``__getstate__`` and ``__setstate__`` that override this behavior.
301 Immutable classes can then implement pickle via ``__reduce__`` or
302 ``__getnewargs__``.
304 Following the example of Python's built-in immutable types, such as `str`
305 and `tuple`, the `immutable` decorator provides a ``__copy__``
306 implementation that just returns ``self``, because there is no reason to
307 actually copy an object if none of its shared owners can modify it.
309 Similarly, objects that are recursively (i.e. are themselves immutable and
310 have only recursively immutable attributes) should also reimplement
311 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as
312 it has no way of checking for recursive immutability.
313 """
314 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807
315 if hasattr(self, name):
316 raise AttributeError(f"{cls.__name__} instances are immutable.")
317 object.__setattr__(self, name, value)
318 # mypy says the variable here has signature (str, Any) i.e. no "self";
319 # I think it's just confused by descriptor stuff.
320 cls.__setattr__ = __setattr__ # type: ignore
322 def __getstate__(self: _T) -> dict: # noqa: N807
323 # Disable default state-setting when unpickled.
324 return {}
325 cls.__getstate__ = __getstate__
327 def __setstate__(self: _T, state: Any) -> None: # noqa: N807
328 # Disable default state-setting when copied.
329 # Sadly what works for pickle doesn't work for copy.
330 assert not state
331 cls.__setstate__ = __setstate__
333 def __copy__(self: _T) -> _T: # noqa: N807
334 return self
335 cls.__copy__ = __copy__
336 return cls
339_S = TypeVar("_S")
340_R = TypeVar("_R")
343def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
344 """A decorator that caches the result of a method that takes only ``self``
345 as an argument, returning the cached result on subsequent calls.
347 Notes
348 -----
349 This is intended primarily as a stopgap for Python 3.8's more sophisticated
350 ``functools.cached_property``, but it is also explicitly compatible with
351 the `immutable` decorator, which may not be true of ``cached_property``.
352 """
353 attribute = f"_cached_{func.__name__}"
355 @functools.wraps(func)
356 def inner(self: _S) -> _R:
357 if not hasattr(self, attribute):
358 object.__setattr__(self, attribute, func(self))
359 return getattr(self, attribute)
361 return inner
364def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
365 """Get the files from a list of values. If a value is a file it is added to
366 the list of returned files. If a value is a directory, all the files in
367 the directory (recursively) that match the regex will be returned.
369 Parameters
370 ----------
371 values : iterable [`str`]
372 The files to return and directories in which to look for files to
373 return.
374 regex : `str`
375 The regex to use when searching for files within directories. Optional,
376 by default returns all the found files.
378 Returns
379 -------
380 resources: `list` [`str`]
381 The passed-in files and files found in passed-in directories.
382 """
383 fileRegex = None if regex is None else re.compile(regex)
384 resources = []
386 # Find all the files of interest
387 for location in values:
388 if os.path.isdir(location):
389 for root, dirs, files in os.walk(location):
390 for name in files:
391 path = os.path.join(root, name)
392 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
393 resources.append(path)
394 else:
395 resources.append(location)
396 return resources
399def globToRegex(expressions: List[str]) -> List[Pattern[str]]:
400 """Translate glob-style search terms to regex.
402 If a stand-alone '*' is found in ``expressions`` then an empty list will be
403 returned, meaning there are no pattern constraints.
405 Parameters
406 ----------
407 expressions : `list` [`str`]
408 A list of glob-style pattern strings to convert.
410 Returns
411 -------
412 expressions : `list` [`str`]
413 A list of regex Patterns.
414 """
415 if "*" in expressions:
416 _LOG.warning("Found a '*' in the glob terms, returning zero search restrictions.")
417 return list()
418 return [re.compile(fnmatch.translate(e)) for e in expressions]