Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "iterable", 

30 "safeMakeDir", 

31 "Singleton", 

32 "stripIfNotNone", 

33 "transactional", 

34) 

35 

36import errno 

37import os 

38import builtins 

39import fnmatch 

40import functools 

41import logging 

42import re 

43from typing import ( 

44 Any, 

45 Callable, 

46 Dict, 

47 Iterable, 

48 Iterator, 

49 List, 

50 Mapping, 

51 Optional, 

52 Pattern, 

53 Type, 

54 TypeVar, 

55 TYPE_CHECKING, 

56 Union, 

57) 

58 

59from lsst.utils import doImport 

60 

61if TYPE_CHECKING: 61 ↛ 62line 61 didn't jump to line 62, because the condition on line 61 was never true

62 from ..registry.wildcards import Ellipsis, EllipsisType 

63 

64 

65_LOG = logging.getLogger(__name__) 

66 

67 

68def safeMakeDir(directory: str) -> None: 

69 """Make a directory in a manner avoiding race conditions.""" 

70 if directory != "" and not os.path.exists(directory): 

71 try: 

72 os.makedirs(directory) 

73 except OSError as e: 

74 # Don't fail if directory exists due to race 

75 if e.errno != errno.EEXIST: 

76 raise e 

77 

78 

79def iterable(a: Any) -> Iterable[Any]: 

80 """Make input iterable. 

81 

82 There are three cases, when the input is: 

83 

84 - iterable, but not a `str` or Mapping -> iterate over elements 

85 (e.g. ``[i for i in a]``) 

86 - a `str` -> return single element iterable (e.g. ``[a]``) 

87 - a Mapping -> return single element iterable 

88 - not iterable -> return single element iterable (e.g. ``[a]``). 

89 

90 Parameters 

91 ---------- 

92 a : iterable or `str` or not iterable 

93 Argument to be converted to an iterable. 

94 

95 Returns 

96 ------- 

97 i : `generator` 

98 Iterable version of the input value. 

99 """ 

100 if isinstance(a, str): 

101 yield a 

102 return 

103 if isinstance(a, Mapping): 

104 yield a 

105 return 

106 try: 

107 yield from a 

108 except Exception: 

109 yield a 

110 

111 

112def allSlots(self: Any) -> Iterator[str]: 

113 """ 

114 Return combined ``__slots__`` for all classes in objects mro. 

115 

116 Parameters 

117 ---------- 

118 self : `object` 

119 Instance to be inspected. 

120 

121 Returns 

122 ------- 

123 slots : `itertools.chain` 

124 All the slots as an iterable. 

125 """ 

126 from itertools import chain 

127 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

128 

129 

130def getFullTypeName(cls: Any) -> str: 

131 """Return full type name of the supplied entity. 

132 

133 Parameters 

134 ---------- 

135 cls : `type` or `object` 

136 Entity from which to obtain the full name. Can be an instance 

137 or a `type`. 

138 

139 Returns 

140 ------- 

141 name : `str` 

142 Full name of type. 

143 

144 Notes 

145 ----- 

146 Builtins are returned without the ``builtins`` specifier included. This 

147 allows `str` to be returned as "str" rather than "builtins.str". Any 

148 parts of the path that start with a leading underscore are removed 

149 on the assumption that they are an implementation detail and the 

150 entity will be hoisted into the parent namespace. 

151 """ 

152 # If we have an instance we need to convert to a type 

153 if not hasattr(cls, "__qualname__"): 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 cls = type(cls) 

155 if hasattr(builtins, cls.__qualname__): 

156 # Special case builtins such as str and dict 

157 return cls.__qualname__ 

158 

159 real_name = cls.__module__ + "." + cls.__qualname__ 

160 

161 # Remove components with leading underscores 

162 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

163 

164 # Consistency check 

165 if real_name != cleaned_name: 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true

166 try: 

167 test = doImport(cleaned_name) 

168 except Exception: 

169 # Could not import anything so return the real name 

170 return real_name 

171 

172 # The thing we imported should match the class we started with 

173 # despite the clean up. If it does not we return the real name 

174 if test is not cls: 

175 return real_name 

176 

177 return cleaned_name 

178 

179 

180def getClassOf(typeOrName: Union[Type, str]) -> Type: 

181 """Given the type name or a type, return the python type. 

182 

183 If a type name is given, an attempt will be made to import the type. 

184 

185 Parameters 

186 ---------- 

187 typeOrName : `str` or Python class 

188 A string describing the Python class to load or a Python type. 

189 

190 Returns 

191 ------- 

192 type_ : `type` 

193 Directly returns the Python type if a type was provided, else 

194 tries to import the given string and returns the resulting type. 

195 

196 Notes 

197 ----- 

198 This is a thin wrapper around `~lsst.utils.doImport`. 

199 """ 

200 if isinstance(typeOrName, str): 

201 cls = doImport(typeOrName) 

202 else: 

203 cls = typeOrName 

204 return cls 

205 

206 

207def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

208 """Given the type name or a type, instantiate an object of that type. 

209 

210 If a type name is given, an attempt will be made to import the type. 

211 

212 Parameters 

213 ---------- 

214 typeOrName : `str` or Python class 

215 A string describing the Python class to load or a Python type. 

216 args : `tuple` 

217 Positional arguments to use pass to the object constructor. 

218 kwargs : `dict` 

219 Keyword arguments to pass to object constructor. 

220 

221 Returns 

222 ------- 

223 instance : `object` 

224 Instance of the requested type, instantiated with the provided 

225 parameters. 

226 """ 

227 cls = getClassOf(typeOrName) 

228 return cls(*args, **kwargs) 

229 

230 

231class Singleton(type): 

232 """Metaclass to convert a class to a Singleton. 

233 

234 If this metaclass is used the constructor for the singleton class must 

235 take no arguments. This is because a singleton class will only accept 

236 the arguments the first time an instance is instantiated. 

237 Therefore since you do not know if the constructor has been called yet it 

238 is safer to always call it with no arguments and then call a method to 

239 adjust state of the singleton. 

240 """ 

241 

242 _instances: Dict[Type, Any] = {} 

243 

244 # Signature is intentionally not substitutable for type.__call__ (no *args, 

245 # **kwargs) to require classes that use this metaclass to have no 

246 # constructor arguments. 

247 def __call__(cls) -> Any: # type: ignore 

248 if cls not in cls._instances: 

249 cls._instances[cls] = super(Singleton, cls).__call__() 

250 return cls._instances[cls] 

251 

252 

253F = TypeVar("F", bound=Callable) 

254 

255 

256def transactional(func: F) -> F: 

257 """Decorate a method and makes it transactional. 

258 

259 This depends on the class also defining a `transaction` method 

260 that takes no arguments and acts as a context manager. 

261 """ 

262 @functools.wraps(func) 

263 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

264 with self.transaction(): 

265 return func(self, *args, **kwargs) 

266 return inner # type: ignore 

267 

268 

269def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

270 """Strip leading and trailing whitespace if the given object is not None. 

271 

272 Parameters 

273 ---------- 

274 s : `str`, optional 

275 Input string. 

276 

277 Returns 

278 ------- 

279 r : `str` or `None` 

280 A string with leading and trailing whitespace stripped if `s` is not 

281 `None`, or `None` if `s` is `None`. 

282 """ 

283 if s is not None: 

284 s = s.strip() 

285 return s 

286 

287 

288_T = TypeVar("_T", bound="Type") 

289 

290 

291def immutable(cls: _T) -> _T: 

292 """Decorate a class to simulates a simple form of immutability. 

293 

294 A class decorated as `immutable` may only set each of its attributes once; 

295 any attempts to set an already-set attribute will raise `AttributeError`. 

296 

297 Notes 

298 ----- 

299 Subclasses of classes marked with ``@immutable`` are also immutable. 

300 

301 Because this behavior interferes with the default implementation for the 

302 ``pickle`` modules, `immutable` provides implementations of 

303 ``__getstate__`` and ``__setstate__`` that override this behavior. 

304 Immutable classes can then implement pickle via ``__reduce__`` or 

305 ``__getnewargs__``. 

306 

307 Following the example of Python's built-in immutable types, such as `str` 

308 and `tuple`, the `immutable` decorator provides a ``__copy__`` 

309 implementation that just returns ``self``, because there is no reason to 

310 actually copy an object if none of its shared owners can modify it. 

311 

312 Similarly, objects that are recursively (i.e. are themselves immutable and 

313 have only recursively immutable attributes) should also reimplement 

314 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as 

315 it has no way of checking for recursive immutability. 

316 """ 

317 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807 

318 if hasattr(self, name): 

319 raise AttributeError(f"{cls.__name__} instances are immutable.") 

320 object.__setattr__(self, name, value) 

321 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

322 # I think it's just confused by descriptor stuff. 

323 cls.__setattr__ = __setattr__ # type: ignore 

324 

325 def __getstate__(self: _T) -> dict: # noqa: N807 

326 # Disable default state-setting when unpickled. 

327 return {} 

328 cls.__getstate__ = __getstate__ 

329 

330 def __setstate__(self: _T, state: Any) -> None: # noqa: N807 

331 # Disable default state-setting when copied. 

332 # Sadly what works for pickle doesn't work for copy. 

333 assert not state 

334 cls.__setstate__ = __setstate__ 

335 

336 def __copy__(self: _T) -> _T: # noqa: N807 

337 return self 

338 cls.__copy__ = __copy__ 

339 return cls 

340 

341 

342_S = TypeVar("_S") 

343_R = TypeVar("_R") 

344 

345 

346def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]: 

347 """Decorate a method to caches the result. 

348 

349 Only works on methods that take only ``self`` 

350 as an argument, and returns the cached result on subsequent calls. 

351 

352 Notes 

353 ----- 

354 This is intended primarily as a stopgap for Python 3.8's more sophisticated 

355 ``functools.cached_property``, but it is also explicitly compatible with 

356 the `immutable` decorator, which may not be true of ``cached_property``. 

357 

358 `cached_getter` guarantees that the cached value will be stored in 

359 an attribute named ``_cached_{name-of-decorated-function}``. Classes that 

360 use `cached_getter` are responsible for guaranteeing that this name is not 

361 otherwise used, and is included if ``__slots__`` is defined. 

362 """ 

363 attribute = f"_cached_{func.__name__}" 

364 

365 @functools.wraps(func) 

366 def inner(self: _S) -> _R: 

367 if not hasattr(self, attribute): 

368 object.__setattr__(self, attribute, func(self)) 

369 return getattr(self, attribute) 

370 

371 return inner 

372 

373 

374def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

375 """Scan the supplied directories and return all matching files. 

376 

377 Get the files from a list of values. If a value is a file it is added to 

378 the list of returned files. If a value is a directory, all the files in 

379 the directory (recursively) that match the regex will be returned. 

380 

381 Parameters 

382 ---------- 

383 values : iterable [`str`] 

384 The files to return and directories in which to look for files to 

385 return. 

386 regex : `str` 

387 The regex to use when searching for files within directories. Optional, 

388 by default returns all the found files. 

389 

390 Returns 

391 ------- 

392 resources: `list` [`str`] 

393 The passed-in files and files found in passed-in directories. 

394 """ 

395 fileRegex = None if regex is None else re.compile(regex) 

396 resources = [] 

397 

398 # Find all the files of interest 

399 for location in values: 

400 if os.path.isdir(location): 

401 for root, dirs, files in os.walk(location): 

402 for name in files: 

403 path = os.path.join(root, name) 

404 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

405 resources.append(path) 

406 else: 

407 resources.append(location) 

408 return resources 

409 

410 

411def globToRegex(expressions: Union[str, EllipsisType, None, 

412 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]: 

413 """Translate glob-style search terms to regex. 

414 

415 If a stand-alone '``*``' is found in ``expressions``, or expressions is 

416 empty or `None`, then the special value ``...`` will be returned, 

417 indicating that any string will match. 

418 

419 Parameters 

420 ---------- 

421 expressions : `str` or `list` [`str`] 

422 A list of glob-style pattern strings to convert. 

423 

424 Returns 

425 ------- 

426 expressions : `list` [`str` or `re.Pattern`] or ``...`` 

427 A list of regex Patterns or simple strings. Returns ``...`` if 

428 the provided expressions would match everything. 

429 """ 

430 if expressions is Ellipsis or expressions is None: 

431 return Ellipsis 

432 expressions = list(iterable(expressions)) 

433 if not expressions or "*" in expressions: 

434 return Ellipsis 

435 

436 nomagic = re.compile(r"^[\w/]+$", re.ASCII) 

437 

438 # Try not to convert simple string to a regex. 

439 results: List[Union[str, Pattern]] = [] 

440 for e in expressions: 

441 res: Union[str, Pattern] 

442 if nomagic.match(e): 

443 res = e 

444 else: 

445 res = re.compile(fnmatch.translate(e)) 

446 results.append(res) 

447 return results