Coverage for python/lsst/daf/butler/core/utils.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "iterable",
30 "safeMakeDir",
31 "Singleton",
32 "stripIfNotNone",
33 "transactional",
34)
36import errno
37import os
38import builtins
39import fnmatch
40import functools
41import logging
42import re
43from typing import (
44 Any,
45 Callable,
46 Dict,
47 Iterable,
48 Iterator,
49 List,
50 Mapping,
51 Optional,
52 Pattern,
53 Type,
54 TypeVar,
55 Union,
56)
58from lsst.utils import doImport
61_LOG = logging.getLogger(__name__.partition(".")[2])
64def safeMakeDir(directory: str) -> None:
65 """Make a directory in a manner avoiding race conditions"""
66 if directory != "" and not os.path.exists(directory):
67 try:
68 os.makedirs(directory)
69 except OSError as e:
70 # Don't fail if directory exists due to race
71 if e.errno != errno.EEXIST:
72 raise e
75def iterable(a: Any) -> Iterable[Any]:
76 """Make input iterable.
78 There are three cases, when the input is:
80 - iterable, but not a `str` or Mapping -> iterate over elements
81 (e.g. ``[i for i in a]``)
82 - a `str` -> return single element iterable (e.g. ``[a]``)
83 - a Mapping -> return single element iterable
84 - not iterable -> return single element iterable (e.g. ``[a]``).
86 Parameters
87 ----------
88 a : iterable or `str` or not iterable
89 Argument to be converted to an iterable.
91 Returns
92 -------
93 i : `generator`
94 Iterable version of the input value.
95 """
96 if isinstance(a, str):
97 yield a
98 return
99 if isinstance(a, Mapping):
100 yield a
101 return
102 try:
103 yield from a
104 except Exception:
105 yield a
108def allSlots(self: Any) -> Iterator[str]:
109 """
110 Return combined ``__slots__`` for all classes in objects mro.
112 Parameters
113 ----------
114 self : `object`
115 Instance to be inspected.
117 Returns
118 -------
119 slots : `itertools.chain`
120 All the slots as an iterable.
121 """
122 from itertools import chain
123 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
126def getFullTypeName(cls: Any) -> str:
127 """Return full type name of the supplied entity.
129 Parameters
130 ----------
131 cls : `type` or `object`
132 Entity from which to obtain the full name. Can be an instance
133 or a `type`.
135 Returns
136 -------
137 name : `str`
138 Full name of type.
140 Notes
141 -----
142 Builtins are returned without the ``builtins`` specifier included. This
143 allows `str` to be returned as "str" rather than "builtins.str". Any
144 parts of the path that start with a leading underscore are removed
145 on the assumption that they are an implementation detail and the
146 entity will be hoisted into the parent namespace.
147 """
148 # If we have an instance we need to convert to a type
149 if not hasattr(cls, "__qualname__"): 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true
150 cls = type(cls)
151 if hasattr(builtins, cls.__qualname__):
152 # Special case builtins such as str and dict
153 return cls.__qualname__
155 real_name = cls.__module__ + "." + cls.__qualname__
157 # Remove components with leading underscores
158 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
160 # Consistency check
161 if real_name != cleaned_name: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true
162 try:
163 test = doImport(cleaned_name)
164 except Exception:
165 # Could not import anything so return the real name
166 return real_name
168 # The thing we imported should match the class we started with
169 # despite the clean up. If it does not we return the real name
170 if test is not cls:
171 return real_name
173 return cleaned_name
176def getClassOf(typeOrName: Union[Type, str]) -> Type:
177 """Given the type name or a type, return the python type.
179 If a type name is given, an attempt will be made to import the type.
181 Parameters
182 ----------
183 typeOrName : `str` or Python class
184 A string describing the Python class to load or a Python type.
186 Returns
187 -------
188 type_ : `type`
189 Directly returns the Python type if a type was provided, else
190 tries to import the given string and returns the resulting type.
192 Notes
193 -----
194 This is a thin wrapper around `~lsst.utils.doImport`.
195 """
196 if isinstance(typeOrName, str):
197 cls = doImport(typeOrName)
198 else:
199 cls = typeOrName
200 return cls
203def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
204 """Given the type name or a type, instantiate an object of that type.
206 If a type name is given, an attempt will be made to import the type.
208 Parameters
209 ----------
210 typeOrName : `str` or Python class
211 A string describing the Python class to load or a Python type.
212 args : `tuple`
213 Positional arguments to use pass to the object constructor.
214 kwargs : `dict`
215 Keyword arguments to pass to object constructor.
217 Returns
218 -------
219 instance : `object`
220 Instance of the requested type, instantiated with the provided
221 parameters.
222 """
223 cls = getClassOf(typeOrName)
224 return cls(*args, **kwargs)
227class Singleton(type):
228 """Metaclass to convert a class to a Singleton.
230 If this metaclass is used the constructor for the singleton class must
231 take no arguments. This is because a singleton class will only accept
232 the arguments the first time an instance is instantiated.
233 Therefore since you do not know if the constructor has been called yet it
234 is safer to always call it with no arguments and then call a method to
235 adjust state of the singleton.
236 """
238 _instances: Dict[Type, Any] = {}
240 # Signature is intentionally not substitutable for type.__call__ (no *args,
241 # **kwargs) to require classes that use this metaclass to have no
242 # constructor arguments.
243 def __call__(cls) -> Any: # type: ignore
244 if cls not in cls._instances:
245 cls._instances[cls] = super(Singleton, cls).__call__()
246 return cls._instances[cls]
249F = TypeVar("F", bound=Callable)
252def transactional(func: F) -> F:
253 """Decorator that wraps a method and makes it transactional.
255 This depends on the class also defining a `transaction` method
256 that takes no arguments and acts as a context manager.
257 """
258 @functools.wraps(func)
259 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
260 with self.transaction():
261 return func(self, *args, **kwargs)
262 return inner # type: ignore
265def stripIfNotNone(s: Optional[str]) -> Optional[str]:
266 """Strip leading and trailing whitespace if the given object is not None.
268 Parameters
269 ----------
270 s : `str`, optional
271 Input string.
273 Returns
274 -------
275 r : `str` or `None`
276 A string with leading and trailing whitespace stripped if `s` is not
277 `None`, or `None` if `s` is `None`.
278 """
279 if s is not None:
280 s = s.strip()
281 return s
284T = TypeVar("T", bound="Type")
287def immutable(cls: T) -> T:
288 """A class decorator that simulates a simple form of immutability for
289 the decorated class.
291 A class decorated as `immutable` may only set each of its attributes once
292 (by convention, in ``__new__``); any attempts to set an already-set
293 attribute will raise `AttributeError`.
295 Because this behavior interferes with the default implementation for
296 the ``pickle`` and ``copy`` modules, `immutable` provides implementations
297 of ``__getstate__`` and ``__setstate__`` that override this behavior.
298 Immutable classes can them implement pickle/copy via ``__getnewargs__``
299 only (other approaches such as ``__reduce__`` and ``__deepcopy__`` may
300 also be used).
301 """
302 def __setattr__(self: Any, name: str, value: Any) -> None: # noqa: N807
303 if hasattr(self, name):
304 raise AttributeError(f"{cls.__name__} instances are immutable.")
305 object.__setattr__(self, name, value)
306 # mypy says the variable here has signature (str, Any) i.e. no "self";
307 # I think it's just confused by descriptor stuff.
308 cls.__setattr__ = __setattr__ # type: ignore
310 def __getstate__(self: Any) -> dict: # noqa: N807
311 # Disable default state-setting when unpickled.
312 return {}
313 cls.__getstate__ = __getstate__
315 def __setstate__(self: Any, state: Any) -> None: # noqa: N807
316 # Disable default state-setting when copied.
317 # Sadly what works for pickle doesn't work for copy.
318 assert not state
319 cls.__setstate__ = __setstate__
320 return cls
323def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
324 """Get the files from a list of values. If a value is a file it is added to
325 the list of returned files. If a value is a directory, all the files in
326 the directory (recursively) that match the regex will be returned.
328 Parameters
329 ----------
330 values : iterable [`str`]
331 The files to return and directories in which to look for files to
332 return.
333 regex : `str`
334 The regex to use when searching for files within directories. Optional,
335 by default returns all the found files.
337 Returns
338 -------
339 resources: `list` [`str`]
340 The passed-in files and files found in passed-in directories.
341 """
342 fileRegex = None if regex is None else re.compile(regex)
343 resources = []
345 # Find all the files of interest
346 for location in values:
347 if os.path.isdir(location):
348 for root, dirs, files in os.walk(location):
349 for name in files:
350 path = os.path.join(root, name)
351 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
352 resources.append(path)
353 else:
354 resources.append(location)
355 return resources
358def globToRegex(expressions: List[str]) -> List[Pattern[str]]:
359 """Translate glob-style search terms to regex.
361 If a stand-alone '*' is found in ``expressions`` then an empty list will be
362 returned, meaning there are no pattern constraints.
364 Parameters
365 ----------
366 expressions : `list` [`str`]
367 A list of glob-style pattern strings to convert.
369 Returns
370 -------
371 expressions : `list` [`str`]
372 A list of regex Patterns.
373 """
374 if "*" in expressions:
375 _LOG.warning("Found a '*' in the glob terms, returning zero search restrictions.")
376 return list()
377 return [re.compile(fnmatch.translate(e)) for e in expressions]