Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "immutable", 

29 "iterable", 

30 "safeMakeDir", 

31 "Singleton", 

32 "stripIfNotNone", 

33 "transactional", 

34) 

35 

36import errno 

37import os 

38import builtins 

39import fnmatch 

40import functools 

41import logging 

42import re 

43from typing import ( 

44 Any, 

45 Callable, 

46 Dict, 

47 Iterable, 

48 Iterator, 

49 List, 

50 Mapping, 

51 Optional, 

52 Pattern, 

53 Type, 

54 TypeVar, 

55 Union, 

56) 

57 

58from lsst.utils import doImport 

59 

60 

61_LOG = logging.getLogger(__name__.partition(".")[2]) 

62 

63 

64def safeMakeDir(directory: str) -> None: 

65 """Make a directory in a manner avoiding race conditions""" 

66 if directory != "" and not os.path.exists(directory): 

67 try: 

68 os.makedirs(directory) 

69 except OSError as e: 

70 # Don't fail if directory exists due to race 

71 if e.errno != errno.EEXIST: 

72 raise e 

73 

74 

75def iterable(a: Any) -> Iterable[Any]: 

76 """Make input iterable. 

77 

78 There are three cases, when the input is: 

79 

80 - iterable, but not a `str` or Mapping -> iterate over elements 

81 (e.g. ``[i for i in a]``) 

82 - a `str` -> return single element iterable (e.g. ``[a]``) 

83 - a Mapping -> return single element iterable 

84 - not iterable -> return single element iterable (e.g. ``[a]``). 

85 

86 Parameters 

87 ---------- 

88 a : iterable or `str` or not iterable 

89 Argument to be converted to an iterable. 

90 

91 Returns 

92 ------- 

93 i : `generator` 

94 Iterable version of the input value. 

95 """ 

96 if isinstance(a, str): 

97 yield a 

98 return 

99 if isinstance(a, Mapping): 

100 yield a 

101 return 

102 try: 

103 yield from a 

104 except Exception: 

105 yield a 

106 

107 

108def allSlots(self: Any) -> Iterator[str]: 

109 """ 

110 Return combined ``__slots__`` for all classes in objects mro. 

111 

112 Parameters 

113 ---------- 

114 self : `object` 

115 Instance to be inspected. 

116 

117 Returns 

118 ------- 

119 slots : `itertools.chain` 

120 All the slots as an iterable. 

121 """ 

122 from itertools import chain 

123 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

124 

125 

126def getFullTypeName(cls: Any) -> str: 

127 """Return full type name of the supplied entity. 

128 

129 Parameters 

130 ---------- 

131 cls : `type` or `object` 

132 Entity from which to obtain the full name. Can be an instance 

133 or a `type`. 

134 

135 Returns 

136 ------- 

137 name : `str` 

138 Full name of type. 

139 

140 Notes 

141 ----- 

142 Builtins are returned without the ``builtins`` specifier included. This 

143 allows `str` to be returned as "str" rather than "builtins.str". Any 

144 parts of the path that start with a leading underscore are removed 

145 on the assumption that they are an implementation detail and the 

146 entity will be hoisted into the parent namespace. 

147 """ 

148 # If we have an instance we need to convert to a type 

149 if not hasattr(cls, "__qualname__"): 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true

150 cls = type(cls) 

151 if hasattr(builtins, cls.__qualname__): 

152 # Special case builtins such as str and dict 

153 return cls.__qualname__ 

154 

155 real_name = cls.__module__ + "." + cls.__qualname__ 

156 

157 # Remove components with leading underscores 

158 cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_")) 

159 

160 # Consistency check 

161 if real_name != cleaned_name: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true

162 try: 

163 test = doImport(cleaned_name) 

164 except Exception: 

165 # Could not import anything so return the real name 

166 return real_name 

167 

168 # The thing we imported should match the class we started with 

169 # despite the clean up. If it does not we return the real name 

170 if test is not cls: 

171 return real_name 

172 

173 return cleaned_name 

174 

175 

176def getClassOf(typeOrName: Union[Type, str]) -> Type: 

177 """Given the type name or a type, return the python type. 

178 

179 If a type name is given, an attempt will be made to import the type. 

180 

181 Parameters 

182 ---------- 

183 typeOrName : `str` or Python class 

184 A string describing the Python class to load or a Python type. 

185 

186 Returns 

187 ------- 

188 type_ : `type` 

189 Directly returns the Python type if a type was provided, else 

190 tries to import the given string and returns the resulting type. 

191 

192 Notes 

193 ----- 

194 This is a thin wrapper around `~lsst.utils.doImport`. 

195 """ 

196 if isinstance(typeOrName, str): 

197 cls = doImport(typeOrName) 

198 else: 

199 cls = typeOrName 

200 return cls 

201 

202 

203def getInstanceOf(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any: 

204 """Given the type name or a type, instantiate an object of that type. 

205 

206 If a type name is given, an attempt will be made to import the type. 

207 

208 Parameters 

209 ---------- 

210 typeOrName : `str` or Python class 

211 A string describing the Python class to load or a Python type. 

212 args : `tuple` 

213 Positional arguments to use pass to the object constructor. 

214 kwargs : `dict` 

215 Keyword arguments to pass to object constructor. 

216 

217 Returns 

218 ------- 

219 instance : `object` 

220 Instance of the requested type, instantiated with the provided 

221 parameters. 

222 """ 

223 cls = getClassOf(typeOrName) 

224 return cls(*args, **kwargs) 

225 

226 

227class Singleton(type): 

228 """Metaclass to convert a class to a Singleton. 

229 

230 If this metaclass is used the constructor for the singleton class must 

231 take no arguments. This is because a singleton class will only accept 

232 the arguments the first time an instance is instantiated. 

233 Therefore since you do not know if the constructor has been called yet it 

234 is safer to always call it with no arguments and then call a method to 

235 adjust state of the singleton. 

236 """ 

237 

238 _instances: Dict[Type, Any] = {} 

239 

240 # Signature is intentionally not substitutable for type.__call__ (no *args, 

241 # **kwargs) to require classes that use this metaclass to have no 

242 # constructor arguments. 

243 def __call__(cls) -> Any: # type: ignore 

244 if cls not in cls._instances: 244 ↛ 246line 244 didn't jump to line 246, because the condition on line 244 was never false

245 cls._instances[cls] = super(Singleton, cls).__call__() 

246 return cls._instances[cls] 

247 

248 

249F = TypeVar("F", bound=Callable) 

250 

251 

252def transactional(func: F) -> F: 

253 """Decorator that wraps a method and makes it transactional. 

254 

255 This depends on the class also defining a `transaction` method 

256 that takes no arguments and acts as a context manager. 

257 """ 

258 @functools.wraps(func) 

259 def inner(self: Any, *args: Any, **kwargs: Any) -> Any: 

260 with self.transaction(): 

261 return func(self, *args, **kwargs) 

262 return inner # type: ignore 

263 

264 

265def stripIfNotNone(s: Optional[str]) -> Optional[str]: 

266 """Strip leading and trailing whitespace if the given object is not None. 

267 

268 Parameters 

269 ---------- 

270 s : `str`, optional 

271 Input string. 

272 

273 Returns 

274 ------- 

275 r : `str` or `None` 

276 A string with leading and trailing whitespace stripped if `s` is not 

277 `None`, or `None` if `s` is `None`. 

278 """ 

279 if s is not None: 

280 s = s.strip() 

281 return s 

282 

283 

284T = TypeVar("T", bound="Type") 

285 

286 

287def immutable(cls: T) -> T: 

288 """A class decorator that simulates a simple form of immutability for 

289 the decorated class. 

290 

291 A class decorated as `immutable` may only set each of its attributes once 

292 (by convention, in ``__new__``); any attempts to set an already-set 

293 attribute will raise `AttributeError`. 

294 

295 Because this behavior interferes with the default implementation for 

296 the ``pickle`` and ``copy`` modules, `immutable` provides implementations 

297 of ``__getstate__`` and ``__setstate__`` that override this behavior. 

298 Immutable classes can them implement pickle/copy via ``__getnewargs__`` 

299 only (other approaches such as ``__reduce__`` and ``__deepcopy__`` may 

300 also be used). 

301 """ 

302 def __setattr__(self: Any, name: str, value: Any) -> None: # noqa: N807 

303 if hasattr(self, name): 

304 raise AttributeError(f"{cls.__name__} instances are immutable.") 

305 object.__setattr__(self, name, value) 

306 # mypy says the variable here has signature (str, Any) i.e. no "self"; 

307 # I think it's just confused by descriptor stuff. 

308 cls.__setattr__ = __setattr__ # type: ignore 

309 

310 def __getstate__(self: Any) -> dict: # noqa: N807 

311 # Disable default state-setting when unpickled. 

312 return {} 

313 cls.__getstate__ = __getstate__ 

314 

315 def __setstate__(self: Any, state: Any) -> None: # noqa: N807 

316 # Disable default state-setting when copied. 

317 # Sadly what works for pickle doesn't work for copy. 

318 assert not state 

319 cls.__setstate__ = __setstate__ 

320 return cls 

321 

322 

323def findFileResources(values: Iterable[str], regex: Optional[str] = None) -> List[str]: 

324 """Get the files from a list of values. If a value is a file it is added to 

325 the list of returned files. If a value is a directory, all the files in 

326 the directory (recursively) that match the regex will be returned. 

327 

328 Parameters 

329 ---------- 

330 values : iterable [`str`] 

331 The files to return and directories in which to look for files to 

332 return. 

333 regex : `str` 

334 The regex to use when searching for files within directories. Optional, 

335 by default returns all the found files. 

336 

337 Returns 

338 ------- 

339 resources: `list` [`str`] 

340 The passed-in files and files found in passed-in directories. 

341 """ 

342 fileRegex = None if regex is None else re.compile(regex) 

343 resources = [] 

344 

345 # Find all the files of interest 

346 for location in values: 

347 if os.path.isdir(location): 

348 for root, dirs, files in os.walk(location): 

349 for name in files: 

350 path = os.path.join(root, name) 

351 if os.path.isfile(path) and (fileRegex is None or fileRegex.search(name)): 

352 resources.append(path) 

353 else: 

354 resources.append(location) 

355 return resources 

356 

357 

358def globToRegex(expressions: List[str]) -> List[Pattern[str]]: 

359 """Translate glob-style search terms to regex. 

360 

361 If a stand-alone '*' is found in ``expressions`` then an empty list will be 

362 returned, meaning there are no pattern constraints. 

363 

364 Parameters 

365 ---------- 

366 expressions : `list` [`str`] 

367 A list of glob-style pattern strings to convert. 

368 

369 Returns 

370 ------- 

371 expressions : `list` [`str`] 

372 A list of regex Patterns. 

373 """ 

374 if "*" in expressions: 

375 _LOG.warning("Found a '*' in the glob terms, returning zero search restrictions.") 

376 return list() 

377 return [re.compile(fnmatch.translate(e)) for e in expressions]