Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "isplit", 

30 "iterable", 

31 "safeMakeDir", 

32 "Singleton", 

33 "stripIfNotNone", 

34 "time_this", 

35 "transactional", 

36) 

37 

38import errno 

39import os 

40import builtins 

41import fnmatch 

42import functools 

43import logging 

44import time 

45import re 

46from contextlib import contextmanager 

47from typing import ( 

48 Any, 

49 Callable, 

50 Dict, 

51 Iterable, 

52 Iterator, 

53 List, 

54 Mapping, 

55 Optional, 

56 Pattern, 

57 Type, 

58 TypeVar, 

59 TYPE_CHECKING, 

60 Union, 

61) 

62 

63from lsst.utils import doImport 

64 

65if TYPE_CHECKING: 65 ↛ 66line 65 didn't jump to line 66, because the condition on line 65 was never true

66 from ..registry.wildcards import Ellipsis, EllipsisType 

67 

68 

69_LOG = logging.getLogger(__name__) 

70 

71 

72def safeMakeDir(directory: str) -> None: 

73 """Make a directory in a manner avoiding race conditions.""" 

74 if directory != "" and not os.path.exists(directory): 

75 try: 

76 os.makedirs(directory) 

77 except OSError as e: 

78 # Don't fail if directory exists due to race 

79 if e.errno != errno.EEXIST: 

80 raise e 

81 

82 

83def iterable(a: Any) -> Iterable[Any]: 

84 """Make input iterable. 

85 

86 There are three cases, when the input is: 

87 

88 - iterable, but not a `str` or Mapping -> iterate over elements 

89 (e.g. ``[i for i in a]``) 

90 - a `str` -> return single element iterable (e.g. ``[a]``) 

91 - a Mapping -> return single element iterable 

92 - not iterable -> return single element iterable (e.g. ``[a]``). 

93 

94 Parameters 

95 ---------- 

96 a : iterable or `str` or not iterable 

97 Argument to be converted to an iterable. 

98 

99 Returns 

100 ------- 

101 i : `generator` 

102 Iterable version of the input value. 

103 """ 

104 if isinstance(a, str): 

105 yield a 

106 return 

107 if isinstance(a, Mapping): 

108 yield a 

109 return 

110 try: 

111 yield from a 

112 except Exception: 

113 yield a 

114 

115 

116def allSlots(self: Any) -> Iterator[str]: 

117 """ 

118 Return combined ``__slots__`` for all classes in objects mro. 

119 

120 Parameters 

121 ---------- 

122 self : `object` 

123 Instance to be inspected. 

124 

125 Returns 

126 ------- 

127 slots : `itertools.chain` 

128 All the slots as an iterable. 

129 """ 

130 from itertools import chain 

131 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

132 

133 

134def getFullTypeName(cls: Any) -> str: 

135 """Return full type name of the supplied entity. 

136 

137 Parameters 

138 ---------- 

139 cls : `type` or `object` 

140 Entity from which to obtain the full name. Can be an instance 

141 or a `type`. 

142 

143 Returns 

144 ------- 

145 name : `str` 

146 Full name of type. 

147 

148 Notes 

149 ----- 

150 Builtins are returned without the ``builtins`` specifier included. This 

151 allows `str` to be returned as "str" rather than "builtins.str". Any 

152 parts of the path that start with a leading underscore are removed 

153 on the assumption that they are an implementation detail and the 

154 entity will be hoisted into the parent namespace. 

155 """ 

156 # If we have an instance we need to convert to a type 

157 if not hasattr(cls, "__qualname__"): 157 ↛ 158line 157 didn't jump to line 158, because the condition on line 157 was never true

158 cls = type(cls) 

159 if hasattr(builtins, cls.__qualname__): 

160 # Special case builtins such as str and dict 

161 return cls.__qualname__ 

162 

163 real_name = cls.__module__ + "." + cls.__qualname__ 

164 

165 # Remove components with leading underscores 

166 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

167 

168 # Consistency check 

169 if real_name != cleaned_name: 169 ↛ 170line 169 didn't jump to line 170, because the condition on line 169 was never true

170 try: 

171 test = doImport(cleaned_name) 

172 except Exception: 

173 # Could not import anything so return the real name 

174 return real_name 

175 

176 # The thing we imported should match the class we started with 

177 # despite the clean up. If it does not we return the real name 

178 if test is not cls: 

179 return real_name 

180 

181 return cleaned_name 

182 

183 

184def getClassOf(typeOrName: Union[Type, str]) -> Type: 

185 """Given the type name or a type, return the python type. 

186 

187 If a type name is given, an attempt will be made to import the type. 

188 

189 Parameters 

190 ---------- 

191 typeOrName : `str` or Python class 

192 A string describing the Python class to load or a Python type. 

193 

194 Returns 

195 ------- 

196 type_ : `type` 

197 Directly returns the Python type if a type was provided, else 

198 tries to import the given string and returns the resulting type. 

199 

200 Notes 

201 ----- 

202 This is a thin wrapper around `~lsst.utils.doImport`. 

203 """ 

204 if isinstance(typeOrName, str): 

205 cls = doImport(typeOrName) 

206 else: 

207 cls = typeOrName 

208 return cls 

209 

210 

211def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

212 """Given the type name or a type, instantiate an object of that type. 

213 

214 If a type name is given, an attempt will be made to import the type. 

215 

216 Parameters 

217 ---------- 

218 typeOrName : `str` or Python class 

219 A string describing the Python class to load or a Python type. 

220 args : `tuple` 

221 Positional arguments to use pass to the object constructor. 

222 **kwargs 

223 Keyword arguments to pass to object constructor. 

224 

225 Returns 

226 ------- 

227 instance : `object` 

228 Instance of the requested type, instantiated with the provided 

229 parameters. 

230 """ 

231 cls = getClassOf(typeOrName) 

232 return cls(*args, **kwargs) 

233 

234 

235class Singleton(type): 

236 """Metaclass to convert a class to a Singleton. 

237 

238 If this metaclass is used the constructor for the singleton class must 

239 take no arguments. This is because a singleton class will only accept 

240 the arguments the first time an instance is instantiated. 

241 Therefore since you do not know if the constructor has been called yet it 

242 is safer to always call it with no arguments and then call a method to 

243 adjust state of the singleton. 

244 """ 

245 

246 _instances: Dict[Type, Any] = {} 

247 

248 # Signature is intentionally not substitutable for type.__call__ (no *args, 

249 # **kwargs) to require classes that use this metaclass to have no 

250 # constructor arguments. 

251 def __call__(cls) -> Any: # type: ignore 

252 if cls not in cls._instances: 

253 cls._instances[cls] = super(Singleton, cls).__call__() 

254 return cls._instances[cls] 

255 

256 

257F = TypeVar("F", bound=Callable) 

258 

259 

260def transactional(func: F) -> F: 

261 """Decorate a method and makes it transactional. 

262 

263 This depends on the class also defining a `transaction` method 

264 that takes no arguments and acts as a context manager. 

265 """ 

266 @functools.wraps(func) 

267 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

268 with self.transaction(): 

269 return func(self, *args, **kwargs) 

270 return inner # type: ignore 

271 

272 

273def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

274 """Strip leading and trailing whitespace if the given object is not None. 

275 

276 Parameters 

277 ---------- 

278 s : `str`, optional 

279 Input string. 

280 

281 Returns 

282 ------- 

283 r : `str` or `None` 

284 A string with leading and trailing whitespace stripped if `s` is not 

285 `None`, or `None` if `s` is `None`. 

286 """ 

287 if s is not None: 

288 s = s.strip() 

289 return s 

290 

291 

292_T = TypeVar("_T", bound="Type") 

293 

294 

295def immutable(cls: _T) -> _T: 

296 """Decorate a class to simulates a simple form of immutability. 

297 

298 A class decorated as `immutable` may only set each of its attributes once; 

299 any attempts to set an already-set attribute will raise `AttributeError`. 

300 

301 Notes 

302 ----- 

303 Subclasses of classes marked with ``@immutable`` are also immutable. 

304 

305 Because this behavior interferes with the default implementation for the 

306 ``pickle`` modules, `immutable` provides implementations of 

307 ``__getstate__`` and ``__setstate__`` that override this behavior. 

308 Immutable classes can then implement pickle via ``__reduce__`` or 

309 ``__getnewargs__``. 

310 

311 Following the example of Python's built-in immutable types, such as `str` 

312 and `tuple`, the `immutable` decorator provides a ``__copy__`` 

313 implementation that just returns ``self``, because there is no reason to 

314 actually copy an object if none of its shared owners can modify it. 

315 

316 Similarly, objects that are recursively (i.e. are themselves immutable and 

317 have only recursively immutable attributes) should also reimplement 

318 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as 

319 it has no way of checking for recursive immutability. 

320 """ 

321 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807 

322 if hasattr(self, name): 

323 raise AttributeError(f"{cls.__name__} instances are immutable.") 

324 object.__setattr__(self, name, value) 

325 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

326 # I think it's just confused by descriptor stuff. 

327 cls.__setattr__ = __setattr__ # type: ignore 

328 

329 def __getstate__(self: _T) -> dict: # noqa: N807 

330 # Disable default state-setting when unpickled. 

331 return {} 

332 cls.__getstate__ = __getstate__ 

333 

334 def __setstate__(self: _T, state: Any) -> None: # noqa: N807 

335 # Disable default state-setting when copied. 

336 # Sadly what works for pickle doesn't work for copy. 

337 assert not state 

338 cls.__setstate__ = __setstate__ 

339 

340 def __copy__(self: _T) -> _T: # noqa: N807 

341 return self 

342 cls.__copy__ = __copy__ 

343 return cls 

344 

345 

346_S = TypeVar("_S") 

347_R = TypeVar("_R") 

348 

349 

350def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]: 

351 """Decorate a method to caches the result. 

352 

353 Only works on methods that take only ``self`` 

354 as an argument, and returns the cached result on subsequent calls. 

355 

356 Notes 

357 ----- 

358 This is intended primarily as a stopgap for Python 3.8's more sophisticated 

359 ``functools.cached_property``, but it is also explicitly compatible with 

360 the `immutable` decorator, which may not be true of ``cached_property``. 

361 

362 `cached_getter` guarantees that the cached value will be stored in 

363 an attribute named ``_cached_{name-of-decorated-function}``. Classes that 

364 use `cached_getter` are responsible for guaranteeing that this name is not 

365 otherwise used, and is included if ``__slots__`` is defined. 

366 """ 

367 attribute = f"_cached_{func.__name__}" 

368 

369 @functools.wraps(func) 

370 def inner(self: _S) -> _R: 

371 if not hasattr(self, attribute): 

372 object.__setattr__(self, attribute, func(self)) 

373 return getattr(self, attribute) 

374 

375 return inner 

376 

377 

378def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

379 """Scan the supplied directories and return all matching files. 

380 

381 Get the files from a list of values. If a value is a file it is added to 

382 the list of returned files. If a value is a directory, all the files in 

383 the directory (recursively) that match the regex will be returned. 

384 

385 Parameters 

386 ---------- 

387 values : iterable [`str`] 

388 The files to return and directories in which to look for files to 

389 return. 

390 regex : `str` 

391 The regex to use when searching for files within directories. Optional, 

392 by default returns all the found files. 

393 

394 Returns 

395 ------- 

396 resources: `list` [`str`] 

397 The passed-in files and files found in passed-in directories. 

398 """ 

399 fileRegex = None if regex is None else re.compile(regex) 

400 resources = [] 

401 

402 # Find all the files of interest 

403 for location in values: 

404 if os.path.isdir(location): 

405 for root, dirs, files in os.walk(location): 

406 for name in files: 

407 path = os.path.join(root, name) 

408 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

409 resources.append(path) 

410 else: 

411 resources.append(location) 

412 return resources 

413 

414 

415def globToRegex(expressions: Union[str, EllipsisType, None, 

416 List[str]]) -> Union[List[Union[str, Pattern]], EllipsisType]: 

417 """Translate glob-style search terms to regex. 

418 

419 If a stand-alone '``*``' is found in ``expressions``, or expressions is 

420 empty or `None`, then the special value ``...`` will be returned, 

421 indicating that any string will match. 

422 

423 Parameters 

424 ---------- 

425 expressions : `str` or `list` [`str`] 

426 A list of glob-style pattern strings to convert. 

427 

428 Returns 

429 ------- 

430 expressions : `list` [`str` or `re.Pattern`] or ``...`` 

431 A list of regex Patterns or simple strings. Returns ``...`` if 

432 the provided expressions would match everything. 

433 """ 

434 if expressions is Ellipsis or expressions is None: 

435 return Ellipsis 

436 expressions = list(iterable(expressions)) 

437 if not expressions or "*" in expressions: 

438 return Ellipsis 

439 

440 nomagic = re.compile(r"^[\w/\.\-]+$", re.ASCII) 

441 

442 # Try not to convert simple string to a regex. 

443 results: List[Union[str, Pattern]] = [] 

444 for e in expressions: 

445 res: Union[str, Pattern] 

446 if nomagic.match(e): 

447 res = e 

448 else: 

449 res = re.compile(fnmatch.translate(e)) 

450 results.append(res) 

451 return results 

452 

453 

454T = TypeVar('T', str, bytes) 

455 

456 

457def isplit(string: T, sep: T) -> Iterator[T]: 

458 """Split a string or bytes by separator returning a generator. 

459 

460 Parameters 

461 ---------- 

462 string : `str` or `bytes` 

463 The string to split into substrings. 

464 sep : `str` or `bytes` 

465 The separator to use to split the string. Must be the same 

466 type as ``string``. Must always be given. 

467 

468 Yields 

469 ------ 

470 subset : `str` or `bytes` 

471 The next subset extracted from the input until the next separator. 

472 """ 

473 begin = 0 

474 while True: 

475 end = string.find(sep, begin) 

476 if end == -1: 

477 yield string[begin:] 

478 return 

479 yield string[begin:end] 

480 begin = end + 1 

481 

482 

483@contextmanager 

484def time_this(log: Optional[logging.Logger] = None, msg: Optional[str] = None, 

485 level: int = logging.DEBUG, prefix: Optional[str] = "timer", 

486 args: Iterable[Any] = ()) -> Iterator[None]: 

487 """Time the enclosed block and issue a log message. 

488 

489 Parameters 

490 ---------- 

491 log : `logging.Logger`, optional 

492 Logger to use to report the timer message. The root logger will 

493 be used if none is given. 

494 msg : `str`, optional 

495 Context to include in log message. 

496 level : `int`, optional 

497 Python logging level to use to issue the log message. If the 

498 code block raises an exception the log message will automatically 

499 switch to level ERROR. 

500 prefix : `str`, optional 

501 Prefix to use to prepend to the supplied logger to 

502 create a new logger to use instead. No prefix is used if the value 

503 is set to `None`. Defaults to "timer". 

504 args : iterable of any 

505 Additional parameters passed to the log command that should be 

506 written to ``msg``. 

507 """ 

508 if log is None: 

509 log = logging.getLogger() 

510 if prefix: 

511 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix 

512 log = logging.getLogger(log_name) 

513 

514 success = False 

515 start = time.time() 

516 try: 

517 yield 

518 success = True 

519 finally: 

520 end = time.time() 

521 

522 # The message is pre-inserted to allow the logger to expand 

523 # the additional args provided. Make that easier by converting 

524 # the None message to empty string. 

525 if msg is None: 

526 msg = "" 

527 

528 if not success: 

529 # Something went wrong so change the log level to indicate 

530 # this. 

531 level = logging.ERROR 

532 

533 # Specify stacklevel to ensure the message is reported from the 

534 # caller (1 is this file, 2 is contextlib, 3 is user) 

535 log.log(level, msg + "%sTook %.4f seconds", *args, 

536 ": " if msg else "", end - start, stacklevel=3)