Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "allSlots", 

25 "getClassOf", 

26 "getFullTypeName", 

27 "getInstanceOf", 

28 "getObjectSize", 

29 "immutable", 

30 "IndexedTupleDict", 

31 "iterable", 

32 "NamedKeyDict", 

33 "NamedValueSet", 

34 "PrivateConstructorMeta", 

35 "Singleton", 

36 "slotValuesAreEqual", 

37 "slotValuesToHash", 

38 "stripIfNotNone", 

39 "transactional", 

40) 

41 

42import builtins 

43import sys 

44import functools 

45from typing import (TypeVar, MutableMapping, Iterator, KeysView, ValuesView, ItemsView, Dict, Union, 

46 MutableSet, Iterable, Mapping, Tuple) 

47from types import MappingProxyType 

48 

49from lsst.utils import doImport 

50 

51 

52def iterable(a): 

53 """Make input iterable. 

54 

55 There are three cases, when the input is: 

56 

57 - iterable, but not a `str` or Mapping -> iterate over elements 

58 (e.g. ``[i for i in a]``) 

59 - a `str` -> return single element iterable (e.g. ``[a]``) 

60 - a Mapping -> return single element iterable 

61 - not iterable -> return single elment iterable (e.g. ``[a]``). 

62 

63 Parameters 

64 ---------- 

65 a : iterable or `str` or not iterable 

66 Argument to be converted to an iterable. 

67 

68 Returns 

69 ------- 

70 i : `generator` 

71 Iterable version of the input value. 

72 """ 

73 if isinstance(a, str): 

74 yield a 

75 return 

76 if isinstance(a, Mapping): 

77 yield a 

78 return 

79 try: 

80 yield from a 

81 except Exception: 

82 yield a 

83 

84 

85def allSlots(self): 

86 """ 

87 Return combined ``__slots__`` for all classes in objects mro. 

88 

89 Parameters 

90 ---------- 

91 self : `object` 

92 Instance to be inspected. 

93 

94 Returns 

95 ------- 

96 slots : `itertools.chain` 

97 All the slots as an iterable. 

98 """ 

99 from itertools import chain 

100 return chain.from_iterable(getattr(cls, "__slots__", []) for cls in self.__class__.__mro__) 

101 

102 

103def slotValuesAreEqual(self, other): 

104 """ 

105 Test for equality by the contents of all slots, including those of its 

106 parents. 

107 

108 Parameters 

109 ---------- 

110 self : `object` 

111 Reference instance. 

112 other : `object` 

113 Comparison instance. 

114 

115 Returns 

116 ------- 

117 equal : `bool` 

118 Returns True if all the slots are equal in both arguments. 

119 """ 

120 return all((getattr(self, slot) == getattr(other, slot) for slot in allSlots(self))) 

121 

122 

123def slotValuesToHash(self): 

124 """ 

125 Generate a hash from slot values. 

126 

127 Parameters 

128 ---------- 

129 self : `object` 

130 Instance to be hashed. 

131 

132 Returns 

133 ------- 

134 h : `int` 

135 Hashed value generated from the slot values. 

136 """ 

137 return hash(tuple(getattr(self, slot) for slot in allSlots(self))) 

138 

139 

140def getFullTypeName(cls): 

141 """Return full type name of the supplied entity. 

142 

143 Parameters 

144 ---------- 

145 cls : `type` or `object` 

146 Entity from which to obtain the full name. Can be an instance 

147 or a `type`. 

148 

149 Returns 

150 ------- 

151 name : `str` 

152 Full name of type. 

153 

154 Notes 

155 ----- 

156 Builtins are returned without the ``builtins`` specifier included. This 

157 allows `str` to be returned as "str" rather than "builtins.str". 

158 """ 

159 # If we have an instance we need to convert to a type 

160 if not hasattr(cls, "__qualname__"): 160 ↛ 161line 160 didn't jump to line 161, because the condition on line 160 was never true

161 cls = type(cls) 

162 if hasattr(builtins, cls.__qualname__): 162 ↛ 164line 162 didn't jump to line 164, because the condition on line 162 was never true

163 # Special case builtins such as str and dict 

164 return cls.__qualname__ 

165 return cls.__module__ + "." + cls.__qualname__ 

166 

167 

168def getClassOf(typeOrName): 

169 """Given the type name or a type, return the python type. 

170 

171 If a type name is given, an attempt will be made to import the type. 

172 

173 Parameters 

174 ---------- 

175 typeOrName : `str` or Python class 

176 A string describing the Python class to load or a Python type. 

177 

178 Returns 

179 ------- 

180 type_ : `type` 

181 Directly returns the Python type if a type was provided, else 

182 tries to import the given string and returns the resulting type. 

183 

184 Notes 

185 ----- 

186 This is a thin wrapper around `~lsst.utils.doImport`. 

187 """ 

188 if isinstance(typeOrName, str): 

189 cls = doImport(typeOrName) 

190 else: 

191 cls = typeOrName 

192 return cls 

193 

194 

195def getInstanceOf(typeOrName, *args, **kwargs): 

196 """Given the type name or a type, instantiate an object of that type. 

197 

198 If a type name is given, an attempt will be made to import the type. 

199 

200 Parameters 

201 ---------- 

202 typeOrName : `str` or Python class 

203 A string describing the Python class to load or a Python type. 

204 args : `tuple` 

205 Positional arguments to use pass to the object constructor. 

206 kwargs : `dict` 

207 Keyword arguments to pass to object constructor. 

208 

209 Returns 

210 ------- 

211 instance : `object` 

212 Instance of the requested type, instantiated with the provided 

213 parameters. 

214 """ 

215 cls = getClassOf(typeOrName) 

216 return cls(*args, **kwargs) 

217 

218 

219class Singleton(type): 

220 """Metaclass to convert a class to a Singleton. 

221 

222 If this metaclass is used the constructor for the singleton class must 

223 take no arguments. This is because a singleton class will only accept 

224 the arguments the first time an instance is instantiated. 

225 Therefore since you do not know if the constructor has been called yet it 

226 is safer to always call it with no arguments and then call a method to 

227 adjust state of the singleton. 

228 """ 

229 

230 _instances = {} 

231 

232 def __call__(cls): 

233 if cls not in cls._instances: 

234 cls._instances[cls] = super(Singleton, cls).__call__() 

235 return cls._instances[cls] 

236 

237 

238def transactional(func): 

239 """Decorator that wraps a method and makes it transactional. 

240 

241 This depends on the class also defining a `transaction` method 

242 that takes no arguments and acts as a context manager. 

243 """ 

244 @functools.wraps(func) 

245 def inner(self, *args, **kwargs): 

246 with self.transaction(): 

247 return func(self, *args, **kwargs) 

248 return inner 

249 

250 

251def getObjectSize(obj, seen=None): 

252 """Recursively finds size of objects. 

253 

254 Only works well for pure python objects. For example it does not work for 

255 ``Exposure`` objects where all the content is behind getter methods. 

256 

257 Parameters 

258 ---------- 

259 obj : `object` 

260 Instance for which size is to be calculated. 

261 seen : `set`, optional 

262 Used internally to keep track of objects already sized during 

263 recursion. 

264 

265 Returns 

266 ------- 

267 size : `int` 

268 Size in bytes. 

269 

270 See Also 

271 -------- 

272 sys.getsizeof 

273 

274 Notes 

275 ----- 

276 See https://goshippo.com/blog/measure-real-size-any-python-object/ 

277 """ 

278 size = sys.getsizeof(obj) 

279 if seen is None: 

280 seen = set() 

281 obj_id = id(obj) 

282 if obj_id in seen: 

283 return 0 

284 # Important mark as seen *before* entering recursion to gracefully handle 

285 # self-referential objects 

286 seen.add(obj_id) 

287 if isinstance(obj, dict): 

288 size += sum([getObjectSize(v, seen) for v in obj.values()]) 

289 size += sum([getObjectSize(k, seen) for k in obj.keys()]) 

290 elif hasattr(obj, "__dict__"): 

291 size += getObjectSize(obj.__dict__, seen) 

292 elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes, bytearray)): 

293 size += sum([getObjectSize(i, seen) for i in obj]) 

294 

295 return size 

296 

297 

298def stripIfNotNone(s): 

299 """Strip leading and trailing whitespace if the given object is not None. 

300 

301 Parameters 

302 ---------- 

303 s : `str`, optional 

304 Input string. 

305 

306 Returns 

307 ------- 

308 r : `str` or `None` 

309 A string with leading and trailing whitespace stripped if `s` is not 

310 `None`, or `None` if `s` is `None`. 

311 """ 

312 if s is not None: 

313 s = s.strip() 

314 return s 

315 

316 

317class PrivateConstructorMeta(type): 

318 """A metaclass that disables regular construction syntax. 

319 

320 A class that uses PrivateConstructorMeta may have an ``__init__`` and/or 

321 ``__new__`` method, but these can't be invoked by "calling" the class 

322 (that will always raise `TypeError`). Instead, such classes can be called 

323 by calling the metaclass-provided `_construct` class method with the same 

324 arguments. 

325 

326 As is usual in Python, there are no actual prohibitions on what code can 

327 call `_construct`; the purpose of this metaclass is just to prevent 

328 instances from being created normally when that can't do what users would 

329 expect. 

330 

331 ..note:: 

332 

333 Classes that inherit from PrivateConstructorMeta also inherit 

334 the hidden-constructor behavior. If you just want to disable 

335 construction of the base class, `abc.ABCMeta` may be a better 

336 option. 

337 

338 Examples 

339 -------- 

340 Given this class definition:: 

341 class Hidden(metaclass=PrivateConstructorMeta): 

342 

343 def __init__(self, a, b): 

344 self.a = a 

345 self.b = b 

346 

347 This doesn't work: 

348 

349 >>> instance = Hidden(a=1, b="two") 

350 TypeError: Hidden objects cannot be constructed directly. 

351 

352 But this does: 

353 

354 >>> instance = Hidden._construct(a=1, b="two") 

355 

356 """ 

357 

358 def __call__(cls, *args, **kwds): 

359 """Disabled class construction interface; always raises `TypeError.` 

360 """ 

361 raise TypeError(f"{cls.__name__} objects cannot be constructed directly.") 

362 

363 def _construct(cls, *args, **kwds): 

364 """Private class construction interface. 

365 

366 All arguments are forwarded to ``__init__`` and/or ``__new__`` 

367 in the usual way. 

368 """ 

369 return type.__call__(cls, *args, **kwds) 

370 

371 

372K = TypeVar("K") 

373V = TypeVar("V") 

374 

375 

376class NamedKeyDict(MutableMapping[K, V]): 

377 """A dictionary wrapper that require keys to have a ``.name`` attribute, 

378 and permits lookups using either key objects or their names. 

379 

380 Names can be used in place of keys when updating existing items, but not 

381 when adding new items. 

382 

383 It is assumed (but asserted) that all name equality is equivalent to key 

384 equality, either because the key objects define equality this way, or 

385 because different objects with the same name are never included in the same 

386 dictionary. 

387 

388 Parameters 

389 ---------- 

390 args 

391 All positional constructor arguments are forwarded directly to `dict`. 

392 Keyword arguments are not accepted, because plain strings are not valid 

393 keys for `NamedKeyDict`. 

394 

395 Raises 

396 ------ 

397 AttributeError 

398 Raised when an attempt is made to add an object with no ``.name`` 

399 attribute to the dictionary. 

400 AssertionError 

401 Raised when multiple keys have the same name. 

402 """ 

403 

404 __slots__ = ("_dict", "_names",) 

405 

406 def __init__(self, *args): 

407 self._dict = dict(*args) 

408 self._names = {key.name: key for key in self._dict} 

409 assert len(self._names) == len(self._dict), "Duplicate names in keys." 

410 

411 @property 

412 def names(self) -> KeysView[str]: 

413 """The set of names associated with the keys, in the same order 

414 (`~collections.abc.KeysView`). 

415 """ 

416 return self._names.keys() 

417 

418 def byName(self) -> Dict[str, V]: 

419 """Return a `dict` with names as keys and the same values as ``self``. 

420 """ 

421 return dict(zip(self._names.keys(), self._dict.values())) 

422 

423 def __len__(self) -> int: 

424 return len(self._dict) 

425 

426 def __iter__(self) -> Iterator[K]: 

427 return iter(self._dict) 

428 

429 def __str__(self) -> str: 

430 return "{{{}}}".format(", ".join(f"{str(k)}: {str(v)}" for k, v in self.items())) 

431 

432 def __repr__(self) -> str: 

433 return "NamedKeyDict({{{}}})".format(", ".join(f"{repr(k)}: {repr(v)}" for k, v in self.items())) 

434 

435 def __getitem__(self, key: Union[str, K]) -> V: 

436 if hasattr(key, "name"): 

437 return self._dict[key] 

438 else: 

439 return self._dict[self._names[key]] 

440 

441 def __setitem__(self, key: Union[str, K], value: V): 

442 if hasattr(key, "name"): 

443 assert self._names.get(key.name, key) == key, "Name is already associated with a different key." 

444 self._dict[key] = value 

445 self._names[key.name] = key 

446 else: 

447 self._dict[self._names[key]] = value 

448 

449 def __delitem__(self, key: Union[str, K]): 

450 if hasattr(key, "name"): 

451 del self._dict[key] 

452 del self._names[key.name] 

453 else: 

454 del self._dict[self._names[key]] 

455 del self._names[key] 

456 

457 def keys(self) -> KeysView[K]: 

458 return self._dict.keys() 

459 

460 def values(self) -> ValuesView[V]: 

461 return self._dict.values() 

462 

463 def items(self) -> ItemsView[K, V]: 

464 return self._dict.items() 

465 

466 def copy(self) -> NamedKeyDict[K, V]: 

467 result = NamedKeyDict.__new__(NamedKeyDict) 

468 result._dict = dict(self._dict) 

469 result._names = dict(self._names) 

470 return result 

471 

472 def freeze(self): 

473 """Disable all mutators, effectively transforming ``self`` into 

474 an immutable mapping. 

475 """ 

476 if not isinstance(self._dict, MappingProxyType): 

477 self._dict = MappingProxyType(self._dict) 

478 

479 

480T = TypeVar("T") 

481 

482 

483class NamedValueSet(MutableSet[T]): 

484 """A custom mutable set class that requires elements to have a ``.name`` 

485 attribute, which can then be used as keys in `dict`-like lookup. 

486 

487 Names and elements can both be used with the ``in`` and ``del`` 

488 operators, `remove`, and `discard`. Names (but not elements) 

489 can be used with ``[]``-based element retrieval (not assignment) 

490 and the `get` method. `pop` can be used in either its `MutableSet` 

491 form (no arguments; an arbitrary element is returned) or its 

492 `MutableMapping` form (one or two arguments for the name and 

493 optional default value, respectively). 

494 

495 Parameters 

496 ---------- 

497 elements : `iterable` 

498 Iterable over elements to include in the set. 

499 

500 Raises 

501 ------ 

502 AttributeError 

503 Raised if one or more elements do not have a ``.name`` attribute. 

504 

505 Notes 

506 ----- 

507 Iteration order is guaranteed to be the same as insertion order (with 

508 the same general behavior as `dict` ordering). 

509 Like `dicts`, sets with the same elements will compare as equal even if 

510 their iterator order is not the same. 

511 """ 

512 

513 __slots__ = ("_dict",) 

514 

515 def __init__(self, elements: Iterable[T] = ()): 

516 self._dict = {element.name: element for element in elements} 

517 

518 @property 

519 def names(self) -> KeysView[str]: 

520 """The set of element names, in the same order 

521 (`~collections.abc.KeysView`). 

522 """ 

523 return self._dict.keys() 

524 

525 def asDict(self) -> Mapping[str, T]: 

526 """Return a mapping view with names as keys. 

527 

528 Returns 

529 ------- 

530 dict : `Mapping` 

531 A dictionary-like view with ``values() == self``. 

532 """ 

533 return self._dict 

534 

535 def __contains__(self, key: Union[str, T]) -> bool: 

536 return getattr(key, "name", key) in self._dict 

537 

538 def __len__(self) -> int: 

539 return len(self._dict) 

540 

541 def __iter__(self) -> Iterator[T]: 

542 return iter(self._dict.values()) 

543 

544 def __str__(self) -> str: 

545 return "{{{}}}".format(", ".join(str(element) for element in self)) 

546 

547 def __repr__(self) -> str: 

548 return "NamedValueSet({{{}}})".format(", ".join(repr(element) for element in self)) 

549 

550 def __eq__(self, other): 

551 try: 

552 return self._dict.keys() == other._dict.keys() 

553 except AttributeError: 

554 return NotImplemented 

555 

556 def __hash__(self): 

557 return hash(frozenset(self._dict.keys())) 

558 

559 # As per Set's docs, overriding just __le__ and __ge__ for performance will 

560 # cover the other comparisons, too. 

561 

562 def __le__(self, other: NamedValueSet[T]) -> bool: 

563 try: 

564 return self._dict.keys() <= other._dict.keys() 

565 except AttributeError: 

566 return NotImplemented 

567 

568 def __ge__(self, other: NamedValueSet[T]) -> bool: 

569 try: 

570 return self._dict.keys() >= other._dict.keys() 

571 except AttributeError: 

572 return NotImplemented 

573 

574 def issubset(self, other): 

575 return self <= other 

576 

577 def issuperset(self, other): 

578 return self >= other 

579 

580 def __getitem__(self, name: str) -> T: 

581 return self._dict[name] 

582 

583 def get(self, name: str, default=None): 

584 """Return the element with the given name, or ``default`` if 

585 no such element is present. 

586 """ 

587 return self._dict.get(name, default) 

588 

589 def __delitem__(self, name: str): 

590 del self._dict[name] 

591 

592 def add(self, element: T): 

593 """Add an element to the set. 

594 

595 Raises 

596 ------ 

597 AttributeError 

598 Raised if the element does not have a ``.name`` attribute. 

599 """ 

600 self._dict[element.name] = element 

601 

602 def remove(self, element: Union[str, T]): 

603 """Remove an element from the set. 

604 

605 Parameters 

606 ---------- 

607 element : `object` or `str` 

608 Element to remove or the string name thereof. Assumed to be an 

609 element if it has a ``.name`` attribute. 

610 

611 Raises 

612 ------ 

613 KeyError 

614 Raised if an element with the given name does not exist. 

615 """ 

616 del self._dict[getattr(element, "name", element)] 

617 

618 def discard(self, element: Union[str, T]): 

619 """Remove an element from the set if it exists. 

620 

621 Does nothing if no matching element is present. 

622 

623 Parameters 

624 ---------- 

625 element : `object` or `str` 

626 Element to remove or the string name thereof. Assumed to be an 

627 element if it has a ``.name`` attribute. 

628 """ 

629 try: 

630 self.remove(element) 

631 except KeyError: 

632 pass 

633 

634 def pop(self, *args): 

635 """Remove and return an element from the set. 

636 

637 Parameters 

638 ---------- 

639 name : `str`, optional 

640 Name of the element to remove and return. Must be passed 

641 positionally. If not provided, an arbitrary element is 

642 removed and returned. 

643 default : `object`, optional 

644 Value to return if ``name`` is provided but no such element 

645 exists. 

646 

647 Raises 

648 ------ 

649 KeyError 

650 Raised if ``name`` is provided but ``default`` is not, and no 

651 matching element exists. 

652 """ 

653 if not args: 

654 return super().pop() 

655 else: 

656 return self._dict.pop(*args) 

657 

658 def copy(self) -> NamedValueSet[T]: 

659 result = NamedValueSet.__new__(NamedValueSet) 

660 result._dict = dict(self._dict) 

661 return result 

662 

663 def freeze(self): 

664 """Disable all mutators, effectively transforming ``self`` into 

665 an immutable set. 

666 """ 

667 if not isinstance(self._dict, MappingProxyType): 

668 self._dict = MappingProxyType(self._dict) 

669 

670 

671class IndexedTupleDict(Mapping[K, V]): 

672 """An immutable mapping that combines a tuple of values with a (possibly 

673 shared) mapping from key to tuple index. 

674 

675 Parameters 

676 ---------- 

677 indices: `~collections.abc.Mapping` 

678 Mapping from key to integer index in the values tuple. This mapping 

679 is used as-is, not copied or converted to a true `dict`, which means 

680 that the caller must guarantee that it will not be modified by other 

681 (shared) owners in the future. If it is a `NamedKeyDict`, both names 

682 and key instances will be usable as keys in the `IndexedTupleDict`. 

683 The caller is also responsible for guaranteeing that the indices in 

684 the mapping are all valid for the given tuple. 

685 values: `tuple` 

686 Tuple of values for the dictionary. The caller is responsible for 

687 guaranteeing that this has the same number of elements as ``indices``. 

688 """ 

689 

690 __slots__ = ("_indices", "_values") 

691 

692 def __new__(cls, indices: Mapping[K, int], values: Tuple[V, ...]): 

693 self = super().__new__(cls) 

694 assert len(indices) == len(values) 

695 self._indices = indices 

696 self._values = values 

697 return self 

698 

699 def __getitem__(self, key: K) -> V: 

700 return self._values[self._indices[key]] 

701 

702 def __iter__(self) -> Iterator[K]: 

703 return iter(self._indices) 

704 

705 def __len__(self) -> int: 

706 return len(self._indices) 

707 

708 def __str__(self) -> str: 

709 return "{{{}}}".format(", ".join(f"{str(k)}: {str(v)}" for k, v in self.items())) 

710 

711 def __repr__(self) -> str: 

712 return "IndexedTupleDict({{{}}})".format(", ".join(f"{repr(k)}: {repr(v)}" for k, v in self.items())) 

713 

714 def __contains__(self, key: K) -> bool: 

715 return key in self._indices 

716 

717 def keys(self) -> KeysView[K]: 

718 return self._indices.keys() 

719 

720 def values(self) -> Tuple[V, ...]: 

721 return self._values 

722 

723 def __getnewargs__(self) -> tuple: 

724 return (self._indices, self._values) 

725 

726 def __getstate__(self) -> dict: # noqa: N807 

727 # Disable default state-setting when unpickled. 

728 return {} 

729 

730 def __setstate__(self, state): # noqa: N807 

731 # Disable default state-setting when copied. 

732 # Sadly what works for pickle doesn't work for copy. 

733 assert not state 

734 

735 # Let Mapping base class provide items(); we can't do it any more 

736 # efficiently ourselves. 

737 

738 

739def immutable(cls): 

740 """A class decorator that simulates a simple form of immutability for 

741 the decorated class. 

742 

743 A class decorated as `immutable` may only set each of its attributes once 

744 (by convention, in ``__new__``); any attempts to set an already-set 

745 attribute will raise `AttributeError`. 

746 

747 Because this behavior interferes with the default implementation for 

748 the ``pickle`` and ``copy`` modules, `immutable` provides implementations 

749 of ``__getstate__`` and ``__setstate__`` that override this behavior. 

750 Immutable classes can them implement pickle/copy via ``__getnewargs__`` 

751 only (other approaches such as ``__reduce__`` and ``__deepcopy__`` may 

752 also be used). 

753 """ 

754 def __setattr__(self, name, value): # noqa: N807 

755 if hasattr(self, name): 

756 raise AttributeError(f"{cls.__name__} instances are immutable.") 

757 object.__setattr__(self, name, value) 

758 cls.__setattr__ = __setattr__ 

759 

760 def __getstate__(self) -> dict: # noqa: N807 

761 # Disable default state-setting when unpickled. 

762 return {} 

763 cls.__getstate__ = __getstate__ 

764 

765 def __setstate__(self, state): # noqa: N807 

766 # Disable default state-setting when copied. 

767 # Sadly what works for pickle doesn't work for copy. 

768 assert not state 

769 cls.__setstate__ = __setstate__ 

770 return cls