Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "isplit", 

30 "iterable", 

31 "safeMakeDir", 

32 "Singleton", 

33 "stripIfNotNone", 

34 "transactional", 

35) 

36 

37import errno 

38import os 

39import builtins 

40import fnmatch 

41import functools 

42import logging 

43import re 

44from typing import ( 

45 Any, 

46 Callable, 

47 Dict, 

48 Iterable, 

49 Iterator, 

50 List, 

51 Mapping, 

52 Optional, 

53 Pattern, 

54 Type, 

55 TypeVar, 

56 TYPE_CHECKING, 

57 Union, 

58) 

59 

60from lsst.utils import doImport 

61 

62if TYPE_CHECKING: 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true

63 from ..registry.wildcards import Ellipsis, EllipsisType 

64 

65 

66_LOG = logging.getLogger(__name__) 

67 

68 

69def safeMakeDir(directory: str) -> None: 

70 """Make a directory in a manner avoiding race conditions.""" 

71 if directory != "" and not os.path.exists(directory): 

72 try: 

73 os.makedirs(directory) 

74 except OSError as e: 

75 # Don't fail if directory exists due to race 

76 if e.errno != errno.EEXIST: 

77 raise e 

78 

79 

80def iterable(a: Any) -> Iterable[Any]: 

81 """Make input iterable. 

82 

83 There are three cases, when the input is: 

84 

85 - iterable, but not a `str` or Mapping -> iterate over elements 

86 (e.g. ``[i for i in a]``) 

87 - a `str` -> return single element iterable (e.g. ``[a]``) 

88 - a Mapping -> return single element iterable 

89 - not iterable -> return single element iterable (e.g. ``[a]``). 

90 

91 Parameters 

92 ---------- 

93 a : iterable or `str` or not iterable 

94 Argument to be converted to an iterable. 

95 

96 Returns 

97 ------- 

98 i : `generator` 

99 Iterable version of the input value. 

100 """ 

101 if isinstance(a, str): 

102 yield a 

103 return 

104 if isinstance(a, Mapping): 

105 yield a 

106 return 

107 try: 

108 yield from a 

109 except Exception: 

110 yield a 

111 

112 

113def allSlots(self: Any) -> Iterator[str]: 

114 """ 

115 Return combined ``__slots__`` for all classes in objects mro. 

116 

117 Parameters 

118 ---------- 

119 self : `object` 

120 Instance to be inspected. 

121 

122 Returns 

123 ------- 

124 slots : `itertools.chain` 

125 All the slots as an iterable. 

126 """ 

127 from itertools import chain 

128 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

129 

130 

131def getFullTypeName(cls: Any) -> str: 

132 """Return full type name of the supplied entity. 

133 

134 Parameters 

135 ---------- 

136 cls : `type` or `object` 

137 Entity from which to obtain the full name. Can be an instance 

138 or a `type`. 

139 

140 Returns 

141 ------- 

142 name : `str` 

143 Full name of type. 

144 

145 Notes 

146 ----- 

147 Builtins are returned without the ``builtins`` specifier included. This 

148 allows `str` to be returned as "str" rather than "builtins.str". Any 

149 parts of the path that start with a leading underscore are removed 

150 on the assumption that they are an implementation detail and the 

151 entity will be hoisted into the parent namespace. 

152 """ 

153 # If we have an instance we need to convert to a type 

154 if not hasattr(cls, "__qualname__"): 154 ↛ 155line 154 didn't jump to line 155, because the condition on line 154 was never true

155 cls = type(cls) 

156 if hasattr(builtins, cls.__qualname__): 

157 # Special case builtins such as str and dict 

158 return cls.__qualname__ 

159 

160 real_name = cls.__module__ + "." + cls.__qualname__ 

161 

162 # Remove components with leading underscores 

163 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

164 

165 # Consistency check 

166 if real_name != cleaned_name: 166 ↛ 167line 166 didn't jump to line 167, because the condition on line 166 was never true

167 try: 

168 test = doImport(cleaned_name) 

169 except Exception: 

170 # Could not import anything so return the real name 

171 return real_name 

172 

173 # The thing we imported should match the class we started with 

174 # despite the clean up. If it does not we return the real name 

175 if test is not cls: 

176 return real_name 

177 

178 return cleaned_name 

179 

180 

181def getClassOf(typeOrName: Union[Type, str]) -> Type: 

182 """Given the type name or a type, return the python type. 

183 

184 If a type name is given, an attempt will be made to import the type. 

185 

186 Parameters 

187 ---------- 

188 typeOrName : `str` or Python class 

189 A string describing the Python class to load or a Python type. 

190 

191 Returns 

192 ------- 

193 type_ : `type` 

194 Directly returns the Python type if a type was provided, else 

195 tries to import the given string and returns the resulting type. 

196 

197 Notes 

198 ----- 

199 This is a thin wrapper around `~lsst.utils.doImport`. 

200 """ 

201 if isinstance(typeOrName, str): 

202 cls = doImport(typeOrName) 

203 else: 

204 cls = typeOrName 

205 return cls 

206 

207 

208def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

209 """Given the type name or a type, instantiate an object of that type. 

210 

211 If a type name is given, an attempt will be made to import the type. 

212 

213 Parameters 

214 ---------- 

215 typeOrName : `str` or Python class 

216 A string describing the Python class to load or a Python type. 

217 args : `tuple` 

218 Positional arguments to use pass to the object constructor. 

219 **kwargs 

220 Keyword arguments to pass to object constructor. 

221 

222 Returns 

223 ------- 

224 instance : `object` 

225 Instance of the requested type, instantiated with the provided 

226 parameters. 

227 """ 

228 cls = getClassOf(typeOrName) 

229 return cls(*args, **kwargs) 

230 

231 

232class Singleton(type): 

233 """Metaclass to convert a class to a Singleton. 

234 

235 If this metaclass is used the constructor for the singleton class must 

236 take no arguments. This is because a singleton class will only accept 

237 the arguments the first time an instance is instantiated. 

238 Therefore since you do not know if the constructor has been called yet it 

239 is safer to always call it with no arguments and then call a method to 

240 adjust state of the singleton. 

241 """ 

242 

243 _instances: Dict[Type, Any] = {} 

244 

245 # Signature is intentionally not substitutable for type.__call__ (no *args, 

246 # **kwargs) to require classes that use this metaclass to have no 

247 # constructor arguments. 

248 def __call__(cls) -> Any: # type: ignore 

249 if cls not in cls._instances: 

250 cls._instances[cls] = super(Singleton, cls).__call__() 

251 return cls._instances[cls] 

252 

253 

254F = TypeVar("F", bound=Callable) 

255 

256 

257def transactional(func: F) -> F: 

258 """Decorate a method and makes it transactional. 

259 

260 This depends on the class also defining a `transaction` method 

261 that takes no arguments and acts as a context manager. 

262 """ 

263 @functools.wraps(func) 

264 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

265 with self.transaction(): 

266 return func(self, *args, **kwargs) 

267 return inner # type: ignore 

268 

269 

270def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

271 """Strip leading and trailing whitespace if the given object is not None. 

272 

273 Parameters 

274 ---------- 

275 s : `str`, optional 

276 Input string. 

277 

278 Returns 

279 ------- 

280 r : `str` or `None` 

281 A string with leading and trailing whitespace stripped if `s` is not 

282 `None`, or `None` if `s` is `None`. 

283 """ 

284 if s is not None: 

285 s = s.strip() 

286 return s 

287 

288 

289_T = TypeVar("_T", bound="Type") 

290 

291 

292def immutable(cls: _T) -> _T: 

293 """Decorate a class to simulates a simple form of immutability. 

294 

295 A class decorated as `immutable` may only set each of its attributes once; 

296 any attempts to set an already-set attribute will raise `AttributeError`. 

297 

298 Notes 

299 ----- 

300 Subclasses of classes marked with ``@immutable`` are also immutable. 

301 

302 Because this behavior interferes with the default implementation for the 

303 ``pickle`` modules, `immutable` provides implementations of 

304 ``__getstate__`` and ``__setstate__`` that override this behavior. 

305 Immutable classes can then implement pickle via ``__reduce__`` or 

306 ``__getnewargs__``. 

307 

308 Following the example of Python's built-in immutable types, such as `str` 

309 and `tuple`, the `immutable` decorator provides a ``__copy__`` 

310 implementation that just returns ``self``, because there is no reason to 

311 actually copy an object if none of its shared owners can modify it. 

312 

313 Similarly, objects that are recursively (i.e. are themselves immutable and 

314 have only recursively immutable attributes) should also reimplement 

315 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as 

316 it has no way of checking for recursive immutability. 

317 """ 

318 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807 

319 if hasattr(self, name): 

320 raise AttributeError(f"{cls.__name__} instances are immutable.") 

321 object.__setattr__(self, name, value) 

322 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

323 # I think it's just confused by descriptor stuff. 

324 cls.__setattr__ = __setattr__ # type: ignore 

325 

326 def __getstate__(self: _T) -> dict: # noqa: N807 

327 # Disable default state-setting when unpickled. 

328 return {} 

329 cls.__getstate__ = __getstate__ 

330 

331 def __setstate__(self: _T, state: Any) -> None: # noqa: N807 

332 # Disable default state-setting when copied. 

333 # Sadly what works for pickle doesn't work for copy. 

334 assert not state 

335 cls.__setstate__ = __setstate__ 

336 

337 def __copy__(self: _T) -> _T: # noqa: N807 

338 return self 

339 cls.__copy__ = __copy__ 

340 return cls 

341 

342 

343_S = TypeVar("_S") 

344_R = TypeVar("_R") 

345 

346 

347def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]: 

348 """Decorate a method to caches the result. 

349 

350 Only works on methods that take only ``self`` 

351 as an argument, and returns the cached result on subsequent calls. 

352 

353 Notes 

354 ----- 

355 This is intended primarily as a stopgap for Python 3.8's more sophisticated 

356 ``functools.cached_property``, but it is also explicitly compatible with 

357 the `immutable` decorator, which may not be true of ``cached_property``. 

358 

359 `cached_getter` guarantees that the cached value will be stored in 

360 an attribute named ``_cached_{name-of-decorated-function}``. Classes that 

361 use `cached_getter` are responsible for guaranteeing that this name is not 

362 otherwise used, and is included if ``__slots__`` is defined. 

363 """ 

364 attribute = f"_cached_{func.__name__}" 

365 

366 @functools.wraps(func) 

367 def inner(self: _S) -> _R: 

368 if not hasattr(self, attribute): 

369 object.__setattr__(self, attribute, func(self)) 

370 return getattr(self, attribute) 

371 

372 return inner 

373 

374 

375def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

376 """Scan the supplied directories and return all matching files. 

377 

378 Get the files from a list of values. If a value is a file it is added to 

379 the list of returned files. If a value is a directory, all the files in 

380 the directory (recursively) that match the regex will be returned. 

381 

382 Parameters 

383 ---------- 

384 values : iterable [`str`] 

385 The files to return and directories in which to look for files to 

386 return. 

387 regex : `str` 

388 The regex to use when searching for files within directories. Optional, 

389 by default returns all the found files. 

390 

391 Returns 

392 ------- 

393 resources: `list` [`str`] 

394 The passed-in files and files found in passed-in directories. 

395 """ 

396 fileRegex = None if regex is None else re.compile(regex) 

397 resources = [] 

398 

399 # Find all the files of interest 

400 for location in values: 

401 if os.path.isdir(location): 

402 for root, dirs, files in os.walk(location): 

403 for name in files: 

404 path = os.path.join(root, name) 

405 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

406 resources.append(path) 

407 else: 

408 resources.append(location) 

409 return resources 

410 

411 

412def globToRegex(expressions: Union[str, EllipsisType, None, 

413 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]: 

414 """Translate glob-style search terms to regex. 

415 

416 If a stand-alone '``*``' is found in ``expressions``, or expressions is 

417 empty or `None`, then the special value ``...`` will be returned, 

418 indicating that any string will match. 

419 

420 Parameters 

421 ---------- 

422 expressions : `str` or `list` [`str`] 

423 A list of glob-style pattern strings to convert. 

424 

425 Returns 

426 ------- 

427 expressions : `list` [`str` or `re.Pattern`] or ``...`` 

428 A list of regex Patterns or simple strings. Returns ``...`` if 

429 the provided expressions would match everything. 

430 """ 

431 if expressions is Ellipsis or expressions is None: 

432 return Ellipsis 

433 expressions = list(iterable(expressions)) 

434 if not expressions or "*" in expressions: 

435 return Ellipsis 

436 

437 nomagic = re.compile(r"^[\w/\.\-]+$", re.ASCII) 

438 

439 # Try not to convert simple string to a regex. 

440 results: List[Union[str, Pattern]] = [] 

441 for e in expressions: 

442 res: Union[str, Pattern] 

443 if nomagic.match(e): 

444 res = e 

445 else: 

446 res = re.compile(fnmatch.translate(e)) 

447 results.append(res) 

448 return results 

449 

450 

451T = TypeVar('T', str, bytes) 

452 

453 

454def isplit(string: T, sep: T) -> Iterator[T]: 

455 """Split a string or bytes by separator returning a generator. 

456 

457 Parameters 

458 ---------- 

459 string : `str` or `bytes` 

460 The string to split into substrings. 

461 sep : `str` or `bytes` 

462 The separator to use to split the string. Must be the same 

463 type as ``string``. Must always be given. 

464 

465 Yields 

466 ------ 

467 subset : `str` or `bytes` 

468 The next subset extracted from the input until the next separator. 

469 """ 

470 begin = 0 

471 while True: 

472 end = string.find(sep, begin) 

473 if end == -1: 

474 yield string[begin:] 

475 return 

476 yield string[begin:end] 

477 begin = end + 1