Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "iterable", 

30 "safeMakeDir", 

31 "Singleton", 

32 "stripIfNotNone", 

33 "transactional", 

34) 

35 

36import errno 

37import os 

38import builtins 

39import fnmatch 

40import functools 

41import logging 

42import re 

43from typing import ( 

44 Any, 

45 Callable, 

46 Dict, 

47 Iterable, 

48 Iterator, 

49 List, 

50 Mapping, 

51 Optional, 

52 Pattern, 

53 Type, 

54 TypeVar, 

55 Union, 

56) 

57 

58from lsst.utils import doImport 

59 

60 

61_LOG = logging.getLogger(__name__.partition(".")[2]) 

62 

63 

64def safeMakeDir(directory: str) -> None: 

65 """Make a directory in a manner avoiding race conditions""" 

66 if directory != "" and not os.path.exists(directory): 

67 try: 

68 os.makedirs(directory) 

69 except OSError as e: 

70 # Don't fail if directory exists due to race 

71 if e.errno != errno.EEXIST: 

72 raise e 

73 

74 

75def iterable(a: Any) -> Iterable[Any]: 

76 """Make input iterable. 

77 

78 There are three cases, when the input is: 

79 

80 - iterable, but not a `str` or Mapping -> iterate over elements 

81 (e.g. ``[i for i in a]``) 

82 - a `str` -> return single element iterable (e.g. ``[a]``) 

83 - a Mapping -> return single element iterable 

84 - not iterable -> return single element iterable (e.g. ``[a]``). 

85 

86 Parameters 

87 ---------- 

88 a : iterable or `str` or not iterable 

89 Argument to be converted to an iterable. 

90 

91 Returns 

92 ------- 

93 i : `generator` 

94 Iterable version of the input value. 

95 """ 

96 if isinstance(a, str): 

97 yield a 

98 return 

99 if isinstance(a, Mapping): 

100 yield a 

101 return 

102 try: 

103 yield from a 

104 except Exception: 

105 yield a 

106 

107 

108def allSlots(self: Any) -> Iterator[str]: 

109 """ 

110 Return combined ``__slots__`` for all classes in objects mro. 

111 

112 Parameters 

113 ---------- 

114 self : `object` 

115 Instance to be inspected. 

116 

117 Returns 

118 ------- 

119 slots : `itertools.chain` 

120 All the slots as an iterable. 

121 """ 

122 from itertools import chain 

123 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

124 

125 

126def getFullTypeName(cls: Any) -> str: 

127 """Return full type name of the supplied entity. 

128 

129 Parameters 

130 ---------- 

131 cls : `type` or `object` 

132 Entity from which to obtain the full name. Can be an instance 

133 or a `type`. 

134 

135 Returns 

136 ------- 

137 name : `str` 

138 Full name of type. 

139 

140 Notes 

141 ----- 

142 Builtins are returned without the ``builtins`` specifier included. This 

143 allows `str` to be returned as "str" rather than "builtins.str". Any 

144 parts of the path that start with a leading underscore are removed 

145 on the assumption that they are an implementation detail and the 

146 entity will be hoisted into the parent namespace. 

147 """ 

148 # If we have an instance we need to convert to a type 

149 if not hasattr(cls, "__qualname__"): 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true

150 cls = type(cls) 

151 if hasattr(builtins, cls.__qualname__): 

152 # Special case builtins such as str and dict 

153 return cls.__qualname__ 

154 

155 real_name = cls.__module__ + "." + cls.__qualname__ 

156 

157 # Remove components with leading underscores 

158 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

159 

160 # Consistency check 

161 if real_name != cleaned_name: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true

162 try: 

163 test = doImport(cleaned_name) 

164 except Exception: 

165 # Could not import anything so return the real name 

166 return real_name 

167 

168 # The thing we imported should match the class we started with 

169 # despite the clean up. If it does not we return the real name 

170 if test is not cls: 

171 return real_name 

172 

173 return cleaned_name 

174 

175 

176def getClassOf(typeOrName: Union[Type, str]) -> Type: 

177 """Given the type name or a type, return the python type. 

178 

179 If a type name is given, an attempt will be made to import the type. 

180 

181 Parameters 

182 ---------- 

183 typeOrName : `str` or Python class 

184 A string describing the Python class to load or a Python type. 

185 

186 Returns 

187 ------- 

188 type_ : `type` 

189 Directly returns the Python type if a type was provided, else 

190 tries to import the given string and returns the resulting type. 

191 

192 Notes 

193 ----- 

194 This is a thin wrapper around `~lsst.utils.doImport`. 

195 """ 

196 if isinstance(typeOrName, str): 

197 cls = doImport(typeOrName) 

198 else: 

199 cls = typeOrName 

200 return cls 

201 

202 

203def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

204 """Given the type name or a type, instantiate an object of that type. 

205 

206 If a type name is given, an attempt will be made to import the type. 

207 

208 Parameters 

209 ---------- 

210 typeOrName : `str` or Python class 

211 A string describing the Python class to load or a Python type. 

212 args : `tuple` 

213 Positional arguments to use pass to the object constructor. 

214 kwargs : `dict` 

215 Keyword arguments to pass to object constructor. 

216 

217 Returns 

218 ------- 

219 instance : `object` 

220 Instance of the requested type, instantiated with the provided 

221 parameters. 

222 """ 

223 cls = getClassOf(typeOrName) 

224 return cls(*args, **kwargs) 

225 

226 

227class Singleton(type): 

228 """Metaclass to convert a class to a Singleton. 

229 

230 If this metaclass is used the constructor for the singleton class must 

231 take no arguments. This is because a singleton class will only accept 

232 the arguments the first time an instance is instantiated. 

233 Therefore since you do not know if the constructor has been called yet it 

234 is safer to always call it with no arguments and then call a method to 

235 adjust state of the singleton. 

236 """ 

237 

238 _instances: Dict[Type, Any] = {} 

239 

240 # Signature is intentionally not substitutable for type.__call__ (no *args, 

241 # **kwargs) to require classes that use this metaclass to have no 

242 # constructor arguments. 

243 def __call__(cls) -> Any: # type: ignore 

244 if cls not in cls._instances: 244 ↛ 246line 244 didn't jump to line 246, because the condition on line 244 was never false

245 cls._instances[cls] = super(Singleton, cls).__call__() 

246 return cls._instances[cls] 

247 

248 

249F = TypeVar("F", bound=Callable) 

250 

251 

252def transactional(func: F) -> F: 

253 """Decorator that wraps a method and makes it transactional. 

254 

255 This depends on the class also defining a `transaction` method 

256 that takes no arguments and acts as a context manager. 

257 """ 

258 @functools.wraps(func) 

259 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

260 with self.transaction(): 

261 return func(self, *args, **kwargs) 

262 return inner # type: ignore 

263 

264 

265def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

266 """Strip leading and trailing whitespace if the given object is not None. 

267 

268 Parameters 

269 ---------- 

270 s : `str`, optional 

271 Input string. 

272 

273 Returns 

274 ------- 

275 r : `str` or `None` 

276 A string with leading and trailing whitespace stripped if `s` is not 

277 `None`, or `None` if `s` is `None`. 

278 """ 

279 if s is not None: 

280 s = s.strip() 

281 return s 

282 

283 

284_T = TypeVar("_T", bound="Type") 

285 

286 

287def immutable(cls: _T) -> _T: 

288 """A class decorator that simulates a simple form of immutability for the 

289 decorated class. 

290 

291 A class decorated as `immutable` may only set each of its attributes once; 

292 any attempts to set an already-set attribute will raise `AttributeError`. 

293 

294 Notes 

295 ----- 

296 Subclasses of classes marked with ``@immutable`` are also immutable. 

297 

298 Because this behavior interferes with the default implementation for the 

299 ``pickle`` modules, `immutable` provides implementations of 

300 ``__getstate__`` and ``__setstate__`` that override this behavior. 

301 Immutable classes can then implement pickle via ``__reduce__`` or 

302 ``__getnewargs__``. 

303 

304 Following the example of Python's built-in immutable types, such as `str` 

305 and `tuple`, the `immutable` decorator provides a ``__copy__`` 

306 implementation that just returns ``self``, because there is no reason to 

307 actually copy an object if none of its shared owners can modify it. 

308 

309 Similarly, objects that are recursively (i.e. are themselves immutable and 

310 have only recursively immutable attributes) should also reimplement 

311 ``__deepcopy__`` to return ``self``. This is not done by the decorator, as 

312 it has no way of checking for recursive immutability. 

313 """ 

314 def __setattr__(self: _T, name: str, value: Any) -> None: # noqa: N807 

315 if hasattr(self, name): 

316 raise AttributeError(f"{cls.__name__} instances are immutable.") 

317 object.__setattr__(self, name, value) 

318 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

319 # I think it's just confused by descriptor stuff. 

320 cls.__setattr__ = __setattr__ # type: ignore 

321 

322 def __getstate__(self: _T) -> dict: # noqa: N807 

323 # Disable default state-setting when unpickled. 

324 return {} 

325 cls.__getstate__ = __getstate__ 

326 

327 def __setstate__(self: _T, state: Any) -> None: # noqa: N807 

328 # Disable default state-setting when copied. 

329 # Sadly what works for pickle doesn't work for copy. 

330 assert not state 

331 cls.__setstate__ = __setstate__ 

332 

333 def __copy__(self: _T) -> _T: # noqa: N807 

334 return self 

335 cls.__copy__ = __copy__ 

336 return cls 

337 

338 

339_S = TypeVar("_S") 

340_R = TypeVar("_R") 

341 

342 

343def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]: 

344 """A decorator that caches the result of a method that takes only ``self`` 

345 as an argument, returning the cached result on subsequent calls. 

346 

347 Notes 

348 ----- 

349 This is intended primarily as a stopgap for Python 3.8's more sophisticated 

350 ``functools.cached_property``, but it is also explicitly compatible with 

351 the `immutable` decorator, which may not be true of ``cached_property``. 

352 """ 

353 attribute = f"_cached_{func.__name__}" 

354 

355 @functools.wraps(func) 

356 def inner(self: _S) -> _R: 

357 if not hasattr(self, attribute): 

358 object.__setattr__(self, attribute, func(self)) 

359 return getattr(self, attribute) 

360 

361 return inner 

362 

363 

364def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

365 """Get the files from a list of values. If a value is a file it is added to 

366 the list of returned files. If a value is a directory, all the files in 

367 the directory (recursively) that match the regex will be returned. 

368 

369 Parameters 

370 ---------- 

371 values : iterable [`str`] 

372 The files to return and directories in which to look for files to 

373 return. 

374 regex : `str` 

375 The regex to use when searching for files within directories. Optional, 

376 by default returns all the found files. 

377 

378 Returns 

379 ------- 

380 resources: `list` [`str`] 

381 The passed-in files and files found in passed-in directories. 

382 """ 

383 fileRegex = None if regex is None else re.compile(regex) 

384 resources = [] 

385 

386 # Find all the files of interest 

387 for location in values: 

388 if os.path.isdir(location): 

389 for root, dirs, files in os.walk(location): 

390 for name in files: 

391 path = os.path.join(root, name) 

392 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

393 resources.append(path) 

394 else: 

395 resources.append(location) 

396 return resources 

397 

398 

399def globToRegex(expressions: List[str]) -> List[Pattern[str]]: 

400 """Translate glob-style search terms to regex. 

401 

402 If a stand-alone '*' is found in ``expressions`` then an empty list will be 

403 returned, meaning there are no pattern constraints. 

404 

405 Parameters 

406 ---------- 

407 expressions : `list` [`str`] 

408 A list of glob-style pattern strings to convert. 

409 

410 Returns 

411 ------- 

412 expressions : `list` [`str`] 

413 A list of regex Patterns. 

414 """ 

415 if "*" in expressions: 

416 _LOG.warning("Found a '*' in the glob terms, returning zero search restrictions.") 

417 return list() 

418 return [re.compile(fnmatch.translate(e)) for e in expressions]