Coverage for python/lsst/daf/butler/core/utils.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "allSlots",
25 "getClassOf",
26 "getFullTypeName",
27 "getInstanceOf",
28 "immutable",
29 "iterable",
30 "safeMakeDir",
31 "Singleton",
32 "stripIfNotNone",
33 "transactional",
34)
36import errno
37import os
38import builtins
39import fnmatch
40import functools
41import logging
42import re
43from typing import (
44 Any,
45 Callable,
46 Dict,
47 Iterable,
48 Iterator,
49 List,
50 Mapping,
51 Optional,
52 Pattern,
53 Type,
54 TypeVar,
55 Union,
56)
58from lsst.utils import doImport
61_LOG = logging.getLogger(__name__.partition(".")[2])
64def safeMakeDir(directory: str) -> None:
65 """Make a directory in a manner avoiding race conditions"""
66 if directory != "" and not os.path.exists(directory):
67 try:
68 os.makedirs(directory)
69 except OSError as e:
70 # Don't fail if directory exists due to race
71 if e.errno != errno.EEXIST:
72 raise e
75def iterable(a: Any) -> Iterable[Any]:
76 """Make input iterable.
78 There are three cases, when the input is:
80 - iterable, but not a `str` or Mapping -> iterate over elements
81 (e.g. ``[i for i in a]``)
82 - a `str` -> return single element iterable (e.g. ``[a]``)
83 - a Mapping -> return single element iterable
84 - not iterable -> return single element iterable (e.g. ``[a]``).
86 Parameters
87 ----------
88 a : iterable or `str` or not iterable
89 Argument to be converted to an iterable.
91 Returns
92 -------
93 i : `generator`
94 Iterable version of the input value.
95 """
96 if isinstance(a, str):
97 yield a
98 return
99 if isinstance(a, Mapping):
100 yield a
101 return
102 try:
103 yield from a
104 except Exception:
105 yield a
108def allSlots(self: Any) -> Iterator[str]:
109 """
110 Return combined ``__slots__`` for all classes in objects mro.
112 Parameters
113 ----------
114 self : `object`
115 Instance to be inspected.
117 Returns
118 -------
119 slots : `itertools.chain`
120 All the slots as an iterable.
121 """
122 from itertools import chain
123 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__)
126def getFullTypeName(cls: Any) -> str:
127 """Return full type name of the supplied entity.
129 Parameters
130 ----------
131 cls : `type` or `object`
132 Entity from which to obtain the full name. Can be an instance
133 or a `type`.
135 Returns
136 -------
137 name : `str`
138 Full name of type.
140 Notes
141 -----
142 Builtins are returned without the ``builtins`` specifier included. This
143 allows `str` to be returned as "str" rather than "builtins.str". Any
144 parts of the path that start with a leading underscore are removed
145 on the assumption that they are an implementation detail and the
146 entity will be hoisted into the parent namespace.
147 """
148 # If we have an instance we need to convert to a type
149 if not hasattr(cls, "__qualname__"): 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true
150 cls = type(cls)
151 if hasattr(builtins, cls.__qualname__):
152 # Special case builtins such as str and dict
153 return cls.__qualname__
155 real_name = cls.__module__ + "." + cls.__qualname__
157 # Remove components with leading underscores
158 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
160 # Consistency check
161 if real_name != cleaned_name: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true
162 try:
163 test = doImport(cleaned_name)
164 except Exception:
165 # Could not import anything so return the real name
166 return real_name
168 # The thing we imported should match the class we started with
169 # despite the clean up. If it does not we return the real name
170 if test is not cls:
171 return real_name
173 return cleaned_name
176def getClassOf(typeOrName: Union[Type, str]) -> Type:
177 """Given the type name or a type, return the python type.
179 If a type name is given, an attempt will be made to import the type.
181 Parameters
182 ----------
183 typeOrName : `str` or Python class
184 A string describing the Python class to load or a Python type.
186 Returns
187 -------
188 type_ : `type`
189 Directly returns the Python type if a type was provided, else
190 tries to import the given string and returns the resulting type.
192 Notes
193 -----
194 This is a thin wrapper around `~lsst.utils.doImport`.
195 """
196 if isinstance(typeOrName, str):
197 cls = doImport(typeOrName)
198 else:
199 cls = typeOrName
200 return cls
203def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
204 """Given the type name or a type, instantiate an object of that type.
206 If a type name is given, an attempt will be made to import the type.
208 Parameters
209 ----------
210 typeOrName : `str` or Python class
211 A string describing the Python class to load or a Python type.
212 args : `tuple`
213 Positional arguments to use pass to the object constructor.
214 kwargs : `dict`
215 Keyword arguments to pass to object constructor.
217 Returns
218 -------
219 instance : `object`
220 Instance of the requested type, instantiated with the provided
221 parameters.
222 """
223 cls = getClassOf(typeOrName)
224 return cls(*args, **kwargs)
227class Singleton(type):
228 """Metaclass to convert a class to a Singleton.
230 If this metaclass is used the constructor for the singleton class must
231 take no arguments. This is because a singleton class will only accept
232 the arguments the first time an instance is instantiated.
233 Therefore since you do not know if the constructor has been called yet it
234 is safer to always call it with no arguments and then call a method to
235 adjust state of the singleton.
236 """
238 _instances: Dict[Type, Any] = {}
240 # Signature is intentionally not substitutable for type.__call__ (no *args,
241 # **kwargs) to require classes that use this metaclass to have no
242 # constructor arguments.
243 def __call__(cls) -> Any: # type: ignore
244 if cls not in cls._instances:
245 cls._instances[cls] = super(Singleton, cls).__call__()
246 return cls._instances[cls]
249F = TypeVar("F", bound=Callable)
252def transactional(func: F) -> F:
253 """Decorator that wraps a method and makes it transactional.
255 This depends on the class also defining a `transaction` method
256 that takes no arguments and acts as a context manager.
257 """
258 @functools.wraps(func)
259 def inner(self: Any, *args: Any, **kwargs: Any) -> Any:
260 with self.transaction():
261 return func(self, *args, **kwargs)
262 return inner # type: ignore
265def stripIfNotNone(s: Optional[str]) -> Optional[str]:
266 """Strip leading and trailing whitespace if the given object is not None.
268 Parameters
269 ----------
270 s : `str`, optional
271 Input string.
273 Returns
274 -------
275 r : `str` or `None`
276 A string with leading and trailing whitespace stripped if `s` is not
277 `None`, or `None` if `s` is `None`.
278 """
279 if s is not None:
280 s = s.strip()
281 return s
284def immutable(cls: Type) -> Type:
285 """A class decorator that simulates a simple form of immutability for
286 the decorated class.
288 A class decorated as `immutable` may only set each of its attributes once
289 (by convention, in ``__new__``); any attempts to set an already-set
290 attribute will raise `AttributeError`.
292 Because this behavior interferes with the default implementation for
293 the ``pickle`` and ``copy`` modules, `immutable` provides implementations
294 of ``__getstate__`` and ``__setstate__`` that override this behavior.
295 Immutable classes can them implement pickle/copy via ``__getnewargs__``
296 only (other approaches such as ``__reduce__`` and ``__deepcopy__`` may
297 also be used).
298 """
299 def __setattr__(self: Any, name: str, value: Any) -> None: # noqa: N807
300 if hasattr(self, name):
301 raise AttributeError(f"{cls.__name__} instances are immutable.")
302 object.__setattr__(self, name, value)
303 # mypy says the variable here has signature (str, Any) i.e. no "self";
304 # I think it's just confused by descriptor stuff.
305 cls.__setattr__ = __setattr__ # type: ignore
307 def __getstate__(self: Any) -> dict: # noqa: N807
308 # Disable default state-setting when unpickled.
309 return {}
310 cls.__getstate__ = __getstate__
312 def __setstate__(self: Any, state: Any) -> None: # noqa: N807
313 # Disable default state-setting when copied.
314 # Sadly what works for pickle doesn't work for copy.
315 assert not state
316 cls.__setstate__ = __setstate__
317 return cls
320def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]:
321 """Get the files from a list of values. If a value is a file it is added to
322 the list of returned files. If a value is a directory, all the files in
323 the directory (recursively) that match the regex will be returned.
325 Parameters
326 ----------
327 values : iterable [`str`]
328 The files to return and directories in which to look for files to
329 return.
330 regex : `str`
331 The regex to use when searching for files within directories. Optional,
332 by default returns all the found files.
334 Returns
335 -------
336 resources: `list` [`str`]
337 The passed-in files and files found in passed-in directories.
338 """
339 fileRegex = None if regex is None else re.compile(regex)
340 resources = []
342 # Find all the files of interest
343 for location in values:
344 if os.path.isdir(location):
345 for root, dirs, files in os.walk(location):
346 for name in files:
347 path = os.path.join(root, name)
348 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)):
349 resources.append(path)
350 else:
351 resources.append(location)
352 return resources
355def globToRegex(expressions: List[str]) -> List[Pattern[str]]:
356 """Translate glob-style search terms to regex.
358 If a stand-alone '*' is found in ``expressions`` then an empty list will be
359 returned, meaning there are no pattern constraints.
361 Parameters
362 ----------
363 expressions : `list` [`str`]
364 A list of glob-style pattern strings to convert.
366 Returns
367 -------
368 expressions : `list` [`str`]
369 A list of regex Patterns.
370 """
371 if "*" in expressions:
372 _LOG.warning("Found a '*' in the glob terms, returning zero search restrictions.")
373 return list()
374 return [re.compile(fnmatch.translate(e)) for e in expressions]