Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "iterable", 

30 "safeMakeDir", 

31 "Singleton", 

32 "stripIfNotNone", 

33 "transactional", 

34) 

35 

36import errno 

37import os 

38import builtins 

39import fnmatch 

40import functools 

41import logging 

42import re 

43from typing import ( 

44 Any, 

45 Callable, 

46 Dict, 

47 Iterable, 

48 Iterator, 

49 List, 

50 Mapping, 

51 Optional, 

52 Pattern, 

53 Type, 

54 TypeVar, 

55 TYPE_CHECKING, 

56 Union, 

57) 

58 

59from lsst.utils import doImport 

60 

61if TYPE_CHECKING: 61 ↛ 62line 61 didn't jump to line 62, because the condition on line 61 was never true

62 from ..registry.wildcards import Ellipsis, EllipsisType 

63 

64 

65_LOG = logging.getLogger(__name__) 

66 

67 

68def safeMakeDir(directory: str) -> None: 

69 """Make a directory in a manner avoiding race conditions""" 

70 if directory != "" and not os.path.exists(directory): 

71 try: 

72 os.makedirs(directory) 

73 except OSError as e: 

74 # Don't fail if directory exists due to race 

75 if e.errno != errno.EEXIST: 

76 raise e 

77 

78 

79def iterable(a: Any) -> Iterable[Any]: 

80 """Make input iterable. 

81 

82 There are three cases, when the input is: 

83 

84 - iterable, but not a `str` or Mapping -> iterate over elements 

85 (e.g. ``[i for i in a]``) 

86 - a `str` -> return single element iterable (e.g. ``[a]``) 

87 - a Mapping -> return single element iterable 

88 - not iterable -> return single element iterable (e.g. ``[a]``). 

89 

90 Parameters 

91 ---------- 

92 a : iterable or `str` or not iterable 

93 Argument to be converted to an iterable. 

94 

95 Returns 

96 ------- 

97 i : `generator` 

98 Iterable version of the input value. 

99 """ 

100 if isinstance(a, str): 

101 yield a 

102 return 

103 if isinstance(a, Mapping): 

104 yield a 

105 return 

106 try: 

107 yield from a 

108 except Exception: 

109 yield a 

110 

111 

112def allSlots(self: Any) -> Iterator[str]: 

113 """ 

114 Return combined ``__slots__`` for all classes in objects mro. 

115 

116 Parameters 

117 ---------- 

118 self : `object` 

119 Instance to be inspected. 

120 

121 Returns 

122 ------- 

123 slots : `itertools.chain` 

124 All the slots as an iterable. 

125 """ 

126 from itertools import chain 

127 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

128 

129 

130def getFullTypeName(cls: Any) -> str: 

131 """Return full type name of the supplied entity. 

132 

133 Parameters 

134 ---------- 

135 cls : `type` or `object` 

136 Entity from which to obtain the full name. Can be an instance 

137 or a `type`. 

138 

139 Returns 

140 ------- 

141 name : `str` 

142 Full name of type. 

143 

144 Notes 

145 ----- 

146 Builtins are returned without the ``builtins`` specifier included. This 

147 allows `str` to be returned as "str" rather than "builtins.str". Any 

148 parts of the path that start with a leading underscore are removed 

149 on the assumption that they are an implementation detail and the 

150 entity will be hoisted into the parent namespace. 

151 """ 

152 # If we have an instance we need to convert to a type 

153 if not hasattr(cls, "__qualname__"): 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 cls = type(cls) 

155 if hasattr(builtins, cls.__qualname__): 

156 # Special case builtins such as str and dict 

157 return cls.__qualname__ 

158 

159 real_name = cls.__module__ + "." + cls.__qualname__ 

160 

161 # Remove components with leading underscores 

162 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

163 

164 # Consistency check 

165 if real_name != cleaned_name: 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true

166 try: 

167 test = doImport(cleaned_name) 

168 except Exception: 

169 # Could not import anything so return the real name 

170 return real_name 

171 

172 # The thing we imported should match the class we started with 

173 # despite the clean up. If it does not we return the real name 

174 if test is not cls: 

175 return real_name 

176 

177 return cleaned_name 

178 

179 

180def getClassOf(typeOrName: Union[Type, str]) -> Type: 

181 """Given the type name or a type, return the python type. 

182 

183 If a type name is given, an attempt will be made to import the type. 

184 

185 Parameters 

186 ---------- 

187 typeOrName : `str` or Python class 

188 A string describing the Python class to load or a Python type. 

189 

190 Returns 

191 ------- 

192 type_ : `type` 

193 Directly returns the Python type if a type was provided, else 

194 tries to import the given string and returns the resulting type. 

195 

196 Notes 

197 ----- 

198 This is a thin wrapper around `~lsst.utils.doImport`. 

199 """ 

200 if isinstance(typeOrName, str): 

201 cls = doImport(typeOrName) 

202 else: 

203 cls = typeOrName 

204 return cls 

205 

206 

207def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

208 """Given the type name or a type, instantiate an object of that type. 

209 

210 If a type name is given, an attempt will be made to import the type. 

211 

212 Parameters 

213 ---------- 

214 typeOrName : `str` or Python class 

215 A string describing the Python class to load or a Python type. 

216 args : `tuple` 

217 Positional arguments to use pass to the object constructor. 

218 kwargs : `dict` 

219 Keyword arguments to pass to object constructor. 

220 

221 Returns 

222 ------- 

223 instance : `object` 

224 Instance of the requested type, instantiated with the provided 

225 parameters. 

226 """ 

227 cls = getClassOf(typeOrName) 

228 return cls(*args, **kwargs) 

229 

230 

231class Singleton(type): 

232 """Metaclass to convert a class to a Singleton. 

233 

234 If this metaclass is used the constructor for the singleton class must 

235 take no arguments. This is because a singleton class will only accept 

236 the arguments the first time an instance is instantiated. 

237 Therefore since you do not know if the constructor has been called yet it 

238 is safer to always call it with no arguments and then call a method to 

239 adjust state of the singleton. 

240 """ 

241 

242 _instances: Dict[Type, Any] = {} 

243 

244 # Signature is intentionally not substitutable for type.__call__ (no *args, 

245 # **kwargs) to require classes that use this metaclass to have no 

246 # constructor arguments. 

247 def __call__(cls) -> Any: # type: ignore 

248 if cls not in cls._instances: 

249 cls._instances[cls] = super(Singleton, cls).__call__() 

250 return cls._instances[cls] 

251 

252 

253F = TypeVar("F", bound=Callable) 

254 

255 

256def transactional(func: F) -> F: 

257 """Decorator that wraps a method and makes it transactional. 

258 

259 This depends on the class also defining a `transaction` method 

260 that takes no arguments and acts as a context manager. 

261 """ 

262 @functools.wraps(func) 

263 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

264 with self.transaction(): 

265 return func(self, *args, **kwargs) 

266 return inner # type: ignore 

267 

268 

269def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

270 """Strip leading and trailing whitespace if the given object is not None. 

271 

272 Parameters 

273 ---------- 

274 s : `str`, optional 

275 Input string. 

276 

277 Returns 

278 ------- 

279 r : `str` or `None` 

280 A string with leading and trailing whitespace stripped if `s` is not 

281 `None`, or `None` if `s` is `None`. 

282 """ 

283 if s is not None: 

284 s = s.strip() 

285 return s 

286 

287 

288_T = TypeVar("_T", bound="Type") 

289 

290 

291def immutable(cls: _T) -> _T: 

292 """A class decorator that simulates a simple form of immutability for the 

293 decorated class. 

294 

295 A class decorated as `immutable` may only set each of its attributes once; 

296 any attempts to set an already-set attribute will raise `AttributeError`. 

297 

298 Notes 

299 ----- 

300 Subclasses of classes marked with ``@immutable`` are also immutable. 

301 

302 Because this behavior interferes with the default implementation for the 

303 ``pickle`` modules, `immutable` provides implementations of 

304 ``__getstate__`` and ``__setstate__`` that override this behavior. 

305 Immutable classes can then implement pickle via ``__reduce__`` or 

306 ``__getnewargs__``. 

307 

308 Following the example of Python's built-in immutable types, such as `str` 

309 and `tuple`, the `immutable` decorator provides a ``__copy__`` 

310 implementation that just returns ``self``, because there is no reason to 

311 actually copy an object if none of its shared owners can modify it. 

312 

313 Similarly, objects that are recursively (i.e. are themselves immutable and 

314 have only recursively immutable attributes) should also reimplement 

315 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as 

316 it has no way of checking for recursive immutability. 

317 """ 

318 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807 

319 if hasattr(self, name): 

320 raise AttributeError(f"{cls.__name__} instances are immutable.") 

321 object.__setattr__(self, name, value) 

322 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

323 # I think it's just confused by descriptor stuff. 

324 cls.__setattr__ = __setattr__ # type: ignore 

325 

326 def __getstate__(self: _T) -> dict: # noqa: N807 

327 # Disable default state-setting when unpickled. 

328 return {} 

329 cls.__getstate__ = __getstate__ 

330 

331 def __setstate__(self: _T, state: Any) -> None: # noqa: N807 

332 # Disable default state-setting when copied. 

333 # Sadly what works for pickle doesn't work for copy. 

334 assert not state 

335 cls.__setstate__ = __setstate__ 

336 

337 def __copy__(self: _T) -> _T: # noqa: N807 

338 return self 

339 cls.__copy__ = __copy__ 

340 return cls 

341 

342 

343_S = TypeVar("_S") 

344_R = TypeVar("_R") 

345 

346 

347def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]: 

348 """A decorator that caches the result of a method that takes only ``self`` 

349 as an argument, returning the cached result on subsequent calls. 

350 

351 Notes 

352 ----- 

353 This is intended primarily as a stopgap for Python 3.8's more sophisticated 

354 ``functools.cached_property``, but it is also explicitly compatible with 

355 the `immutable` decorator, which may not be true of ``cached_property``. 

356 

357 `cached_getter` guarantees that the cached value will be stored in 

358 an attribute named ``_cached_{name-of-decorated-function}``. Classes that 

359 use `cached_getter` are responsible for guaranteeing that this name is not 

360 otherwise used, and is included if ``__slots__`` is defined. 

361 """ 

362 attribute = f"_cached_{func.__name__}" 

363 

364 @functools.wraps(func) 

365 def inner(self: _S) -> _R: 

366 if not hasattr(self, attribute): 

367 object.__setattr__(self, attribute, func(self)) 

368 return getattr(self, attribute) 

369 

370 return inner 

371 

372 

373def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

374 """Get the files from a list of values. If a value is a file it is added to 

375 the list of returned files. If a value is a directory, all the files in 

376 the directory (recursively) that match the regex will be returned. 

377 

378 Parameters 

379 ---------- 

380 values : iterable [`str`] 

381 The files to return and directories in which to look for files to 

382 return. 

383 regex : `str` 

384 The regex to use when searching for files within directories. Optional, 

385 by default returns all the found files. 

386 

387 Returns 

388 ------- 

389 resources: `list` [`str`] 

390 The passed-in files and files found in passed-in directories. 

391 """ 

392 fileRegex = None if regex is None else re.compile(regex) 

393 resources = [] 

394 

395 # Find all the files of interest 

396 for location in values: 

397 if os.path.isdir(location): 

398 for root, dirs, files in os.walk(location): 

399 for name in files: 

400 path = os.path.join(root, name) 

401 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

402 resources.append(path) 

403 else: 

404 resources.append(location) 

405 return resources 

406 

407 

408def globToRegex(expressions: List[str]) -> Union[List[Pattern[str]], EllipsisType]: 

409 """Translate glob-style search terms to regex. 

410 

411 If a stand-alone '*' is found in ``expressions``, or expressions is empty, 

412 then the special value ``...`` will be returned, indicating that any string 

413 will match. 

414 

415 Parameters 

416 ---------- 

417 expressions : `list` [`str`] 

418 A list of glob-style pattern strings to convert. 

419 

420 Returns 

421 ------- 

422 expressions : `list` [`str`] or ``...`` 

423 A list of regex Patterns 

424 """ 

425 if not expressions or "*" in expressions: 

426 return Ellipsis 

427 return [re.compile(fnmatch.translate(e)) for e in expressions]