lsst.daf.base  17.0.1-2-g3bdf598
propertyContainerContinued.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 
25 __all__ = ["getPropertySetState", "getPropertyListState", "setPropertySetState", "setPropertyListState"]
26 
27 import enum
28 import numbers
29 import warnings
30 from collections.abc import Mapping, KeysView
31 
32 # Ensure that C++ exceptions are properly translated to Python
33 import lsst.pex.exceptions # noqa: F401
34 from lsst.utils import continueClass
35 
36 from .propertySet import PropertySet
37 from .propertyList import PropertyList
38 from ..dateTime import DateTime
39 
40 
41 def getPropertySetState(container, asLists=False):
42  """Get the state of a PropertySet in a form that can be pickled.
43 
44  Parameters
45  ----------
46  container : `PropertySet`
47  The property container.
48  asLists : `bool`, optional
49  If False, the default, `tuple` will be used for the contents. If true
50  a `list` will be used.
51 
52  Returns
53  -------
54  state : `list` of `tuple` or `list` of `list`
55  The state, as a list of tuples (or lists), each of which contains
56  the following 3 items:
57  - name (a `str`): the name of the item
58  - elementTypeName (a `str`): the suffix of a ``setX`` method name
59  which is appropriate for the data type. For example integer
60  data has ``elementTypeName="Int"` which corresponds to
61  the ``setInt`` method.
62  - value: the data for the item, in a form compatible
63  with the set method named by ``elementTypeName``
64  """
65  names = container.names(topLevelOnly=True)
66  sequence = list if asLists else tuple
67  return [sequence((name, _propertyContainerElementTypeName(container, name),
68  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO)))
69  for name in names]
70 
71 
72 def getPropertyListState(container, asLists=False):
73  """Get the state of a PropertyList in a form that can be pickled.
74 
75  Parameters
76  ----------
77  container : `PropertyList`
78  The property container.
79  asLists : `bool`, optional
80  If False, the default, `tuple` will be used for the contents. If true
81  a `list` will be used.
82 
83  Returns
84  -------
85  state : `list` of `tuple` or `list` of `list`
86  The state, as a list of tuples (or lists), each of which contains
87  the following 4 items:
88  - name (a `str`): the name of the item
89  - elementTypeName (a `str`): the suffix of a ``setX`` method name
90  which is appropriate for the data type. For example integer
91  data has ``elementTypeName="Int"` which corresponds to
92  the ``setInt`` method.
93  - value: the data for the item, in a form compatible
94  with the set method named by ``elementTypeName``
95  - comment (a `str`): the comment. This item is only present
96  if ``container`` is a PropertyList.
97  """
98  sequence = list if asLists else tuple
99  return [sequence((name, _propertyContainerElementTypeName(container, name),
100  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO),
101  container.getComment(name)))
102  for name in container.getOrderedNames()]
103 
104 
105 def setPropertySetState(container, state):
106  """Restore the state of a PropertySet, in place.
107 
108  Parameters
109  ----------
110  container : `PropertySet`
111  The property container whose state is to be restored.
112  It should be empty to start with and is updated in place.
113  state : `list`
114  The state, as returned by `getPropertySetState`
115  """
116  for name, elemType, value in state:
117  if elemType is not None:
118  getattr(container, "set" + elemType)(name, value)
119  else:
120  raise ValueError(f"Unrecognized values for state restoration: ({name}, {elemType}, {value})")
121 
122 
123 def setPropertyListState(container, state):
124  """Restore the state of a PropertyList, in place.
125 
126  Parameters
127  ----------
128  container : `PropertyList`
129  The property container whose state is to be restored.
130  It should be empty to start with and is updated in place.
131  state : `list`
132  The state, as returned by ``getPropertyListState``
133  """
134  for name, elemType, value, comment in state:
135  getattr(container, "set" + elemType)(name, value, comment)
136 
137 
138 class ReturnStyle(enum.Enum):
139  ARRAY = enum.auto()
140  SCALAR = enum.auto()
141  AUTO = enum.auto()
142 
143 
144 def _propertyContainerElementTypeName(container, name):
145  """Return name of the type of a particular element"""
146  try:
147  t = container.typeOf(name)
148  except LookupError as e:
149  # KeyError is more commonly expected when asking for an element
150  # from a mapping.
151  raise KeyError(str(e))
152  for checkType in ("Bool", "Short", "Int", "Long", "LongLong", "Float", "Double", "String", "DateTime",
153  "PropertySet", "Undef"):
154  if t == getattr(container, "TYPE_" + checkType):
155  return checkType
156  return None
157 
158 
159 def _propertyContainerGet(container, name, returnStyle):
160  """Get a value of unknown type as a scalar or array
161 
162  Parameters
163  ----------
164  container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`
165  Container from which to get the value
166  name : `str`
167  Name of item
168  returnStyle : `ReturnStyle`
169  Control whether numeric or string data is returned as an array
170  or scalar (the other types, ``PropertyList``, ``PropertySet``
171  and ``PersistablePtr``, are always returned as a scalar):
172  - ReturnStyle.ARRAY: return numeric or string data types
173  as an array of values.
174  - ReturnStyle.SCALAR: return numeric or string data types
175  as a single value; if the item has multiple values then
176  return the last value.
177  - ReturnStyle.AUTO: (deprecated) return numeric or string data
178  as a scalar if there is just one item, or as an array
179  otherwise.
180 
181  Raises
182  ------
183  KeyError
184  The specified key does not exist in the container.
185  TypeError
186  The value retrieved is of an unexpected type.
187  ValueError
188  The value for ``returnStyle`` is not correct.
189  """
190  if not container.exists(name):
191  raise KeyError(name + " not found")
192  if returnStyle not in ReturnStyle:
193  raise ValueError("returnStyle {} must be a ReturnStyle".format(returnStyle))
194 
195  elemType = _propertyContainerElementTypeName(container, name)
196  if elemType and elemType != "PropertySet":
197  value = getattr(container, "getArray" + elemType)(name)
198  if returnStyle == ReturnStyle.ARRAY or (returnStyle == ReturnStyle.AUTO and len(value) > 1):
199  return value
200  return value[-1]
201 
202  if container.isPropertySetPtr(name):
203  try:
204  return container.getAsPropertyListPtr(name)
205  except Exception:
206  return container.getAsPropertySetPtr(name)
207  try:
208  return container.getAsPersistablePtr(name)
209  except Exception:
210  pass
211  raise TypeError('Unknown PropertySet value type for ' + name)
212 
213 
214 def _guessIntegerType(container, name, value):
215  """Given an existing container and name, determine the type
216  that should be used for the supplied value. The supplied value
217  is assumed to be a scalar.
218 
219  On Python 3 all ints are LongLong but we need to be able to store them
220  in Int containers if that is what is being used (testing for truncation).
221  Int is assumed to mean 32bit integer (2147483647 to -2147483648).
222 
223  If there is no pre-existing value we have to decide what to do. For now
224  we pick Int if the value is less than maxsize.
225 
226  Returns None if the value supplied is a bool or not an integral value.
227  """
228  useType = None
229  maxInt = 2147483647
230  minInt = -2147483648
231 
232  # We do not want to convert bool to int so let the system work that
233  # out itself
234  if isinstance(value, bool):
235  return useType
236 
237  if isinstance(value, numbers.Integral):
238  try:
239  containerType = _propertyContainerElementTypeName(container, name)
240  except LookupError:
241  # nothing in the container so choose based on size. Safe option is to
242  # always use LongLong
243  if value <= maxInt and value >= minInt:
244  useType = "Int"
245  else:
246  useType = "LongLong"
247  else:
248  if containerType == "Int":
249  # Always use an Int even if we know it won't fit. The later
250  # code will trigger OverflowError if appropriate. Setting the
251  # type to LongLong here will trigger a TypeError instead so it's
252  # best to trigger a predictable OverflowError.
253  useType = "Int"
254  elif containerType == "LongLong":
255  useType = "LongLong"
256  return useType
257 
258 
259 def _propertyContainerSet(container, name, value, typeMenu, *args):
260  """Set a single Python value of unknown type"""
261  if hasattr(value, "__iter__") and not isinstance(value, (str, PropertySet, PropertyList)):
262  exemplar = value[0]
263  else:
264  exemplar = value
265 
266  t = type(exemplar)
267  setType = _guessIntegerType(container, name, exemplar)
268 
269  if setType is not None or t in typeMenu:
270  if setType is None:
271  setType = typeMenu[t]
272  return getattr(container, "set" + setType)(name, value, *args)
273  # Allow for subclasses
274  for checkType in typeMenu:
275  if (checkType is None and exemplar is None) or \
276  (checkType is not None and isinstance(exemplar, checkType)):
277  return getattr(container, "set" + typeMenu[checkType])(name, value, *args)
278  raise TypeError("Unknown value type for key '%s': %s" % (name, t))
279 
280 
281 def _propertyContainerAdd(container, name, value, typeMenu, *args):
282  """Add a single Python value of unknown type"""
283  if hasattr(value, "__iter__"):
284  exemplar = value[0]
285  else:
286  exemplar = value
287 
288  t = type(exemplar)
289  addType = _guessIntegerType(container, name, exemplar)
290 
291  if addType is not None or t in typeMenu:
292  if addType is None:
293  addType = typeMenu[t]
294  return getattr(container, "add" + addType)(name, value, *args)
295  # Allow for subclasses
296  for checkType in typeMenu:
297  if (checkType is None and exemplar is None) or \
298  (checkType is not None and isinstance(exemplar, checkType)):
299  return getattr(container, "add" + typeMenu[checkType])(name, value, *args)
300  raise TypeError("Unknown value type for key '%s': %s" % (name, t))
301 
302 
303 def _makePropertySet(state):
304  """Make a `PropertySet` from the state returned by `getPropertySetState`
305 
306  Parameters
307  ----------
308  state : `list`
309  The data returned by `getPropertySetState`.
310  """
311  ps = PropertySet()
312  setPropertySetState(ps, state)
313  return ps
314 
315 
316 def _makePropertyList(state):
317  """Make a `PropertyList` from the state returned by
318  `getPropertyListState`
319 
320  Parameters
321  ----------
322  state : `list`
323  The data returned by `getPropertySetState`.
324  """
325  pl = PropertyList()
326  setPropertyListState(pl, state)
327  return pl
328 
329 
330 @continueClass
332  # Mapping of type to method names;
333  # int types are omitted due to use of _guessIntegerType
334  _typeMenu = {bool: "Bool",
335  float: "Double",
336  str: "String",
337  DateTime: "DateTime",
338  PropertySet: "PropertySet",
339  PropertyList: "PropertySet",
340  None: "Undef",
341  }
342 
343  def get(self, name):
344  """Return an item as a scalar or array
345 
346  Return an array if the item is of numeric or string type and has
347  more than one value, otherwise return a scalar.
348 
349  .. deprecated:: 20180-06
350  `get` is superseded by `getArray` or `getScalar`
351 
352  Parameters
353  ----------
354  name : ``str``
355  Name of item
356 
357  Raises
358  ------
359  KeyError
360  If the item does not exist.
361  """
362  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
363  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
364 
365  def getArray(self, name):
366  """Return an item as an array if the item is numeric or string
367 
368  If the item is a `PropertySet`, `PropertyList` or
369  `lsst.daf.base.PersistablePtr` then return the item as a scalar.
370 
371  Parameters
372  ----------
373  name : `str`
374  Name of item
375 
376  Raises
377  ------
378  KeyError
379  If the item does not exist.
380  """
381  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
382 
383  def getScalar(self, name):
384  """Return an item as a scalar
385 
386  If the item has more than one value then the last value is returned
387 
388  Parameters
389  ----------
390  name : `str`
391  Name of item
392 
393  Raises
394  ------
395  KeyError
396  If the item does not exist.
397  """
398  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
399 
400  def set(self, name, value):
401  """Set the value of an item
402 
403  If the item already exists it is silently replaced; the types
404  need not match.
405 
406  Parameters
407  ----------
408  name : `str`
409  Name of item
410  value : any supported type
411  Value of item; may be a scalar or array
412  """
413  return _propertyContainerSet(self, name, value, self._typeMenu)
414 
415  def add(self, name, value):
416  """Append one or more values to a given item, which need not exist
417 
418  If the item exists then the new value(s) are appended;
419  otherwise it is like calling `set`
420 
421  Parameters
422  ----------
423  name : `str`
424  Name of item
425  value : any supported type
426  Value of item; may be a scalar or array
427 
428  Notes
429  -----
430  If ``value`` is an `lsst.daf.base.PropertySet` or
431  `lsst.daf.base.PropertyList` then ``value`` replaces
432  the existing value. Also the item is added as a live
433  reference, so updating ``value`` will update this container
434  and vice-versa.
435 
436  Raises
437  ------
438  lsst::pex::exceptions::TypeError
439  If the type of `value` is incompatible with the existing value
440  of the item.
441  """
442  return _propertyContainerAdd(self, name, value, self._typeMenu)
443 
444  def toDict(self):
445  """Returns a (possibly nested) dictionary with all properties.
446 
447  Returns
448  -------
449  d : `dict`
450  Dictionary with all names and values (no comments).
451  """
452 
453  d = {}
454  for name in self.names():
455  v = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
456 
457  if isinstance(v, PropertySet):
458  d[name] = PropertySet.toDict(v)
459  else:
460  d[name] = v
461  return d
462 
463  def __eq__(self, other):
464  if type(self) != type(other):
465  return False
466 
467  if len(self) != len(other):
468  return False
469 
470  for name in self:
471  if _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO) != \
472  _propertyContainerGet(other, name, returnStyle=ReturnStyle.AUTO):
473  return False
474  if self.typeOf(name) != other.typeOf(name):
475  return False
476 
477  return True
478 
479  def __copy__(self):
480  # Copy without having to go through pickle state
481  ps = PropertySet()
482  for itemName in self:
483  ps.copy(itemName, self, itemName)
484  return ps
485 
486  def __deepcopy__(self, memo):
487  result = self.deepCopy()
488  memo[id(self)] = result
489  return result
490 
491  def __contains__(self, name):
492  # Do not use exists() because that includes "."-delimited names
493  return name in self.names(topLevelOnly=True)
494 
495  def __setitem__(self, name, value):
496  if isinstance(value, Mapping):
497  # Create a property set instead
498  ps = PropertySet()
499  for k, v in value.items():
500  ps[k] = v
501  value = ps
502  self.set(name, value)
503 
504  def __delitem__(self, name):
505  if name in self:
506  self.remove(name)
507  else:
508  raise KeyError(f"{name} not present in dict")
509 
510  def __str__(self):
511  return self.toString()
512 
513  def __len__(self):
514  return self.nameCount(topLevelOnly=True)
515 
516  def __iter__(self):
517  for n in self.names(topLevelOnly=True):
518  yield n
519 
520  def keys(self):
521  return KeysView(self)
522 
523  def __reduce__(self):
524  # It would be a bit simpler to use __setstate__ and __getstate__.
525  # However, implementing __setstate__ in Python causes segfaults
526  # because pickle creates a new instance by calling
527  # object.__new__(PropertyList, *args) which bypasses
528  # the pybind11 memory allocation step.
529  return (_makePropertySet, (getPropertySetState(self),))
530 
531 
532 @continueClass
534  # Mapping of type to method names
535  _typeMenu = {bool: "Bool",
536  int: "Int",
537  float: "Double",
538  str: "String",
539  DateTime: "DateTime",
540  PropertySet: "PropertySet",
541  PropertyList: "PropertySet",
542  None: "Undef",
543  }
544 
545  COMMENTSUFFIX = "#COMMENT"
546 
547  def get(self, name):
548  """Return an item as a scalar or array
549 
550  Return an array if the item has more than one value,
551  otherwise return a scalar.
552 
553  .. deprecated:: 20180-06
554  `get` is superseded by `getArray` or `getScalar`
555 
556  Parameters
557  ----------
558  name : `str`
559  Name of item
560 
561  Raises
562  ------
563  KeyError
564  If the item does not exist.
565  """
566  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
567  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
568 
569  def getArray(self, name):
570  """Return an item as an array
571 
572  Parameters
573  ----------
574  name : `str`
575  Name of item
576 
577  Raises
578  ------
579  KeyError
580  If the item does not exist.
581  """
582  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
583 
584  def getScalar(self, name):
585  """Return an item as a scalar
586 
587  If the item has more than one value then the last value is returned
588 
589  Parameters
590  ----------
591  name : `str`
592  Name of item
593 
594  Raises
595  ------
596  KeyError
597  If the item does not exist.
598  """
599  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
600 
601  def set(self, name, value, comment=None):
602  """Set the value of an item
603 
604  If the item already exists it is silently replaced; the types
605  need not match.
606 
607  Parameters
608  ----------
609  name : `str`
610  Name of item
611  value : any supported type
612  Value of item; may be a scalar or array
613  """
614  args = []
615  if comment is not None:
616  args.append(comment)
617  return _propertyContainerSet(self, name, value, self._typeMenu, *args)
618 
619  def add(self, name, value, comment=None):
620  """Append one or more values to a given item, which need not exist
621 
622  If the item exists then the new value(s) are appended;
623  otherwise it is like calling `set`
624 
625  Parameters
626  ----------
627  name : `str`
628  Name of item
629  value : any supported type
630  Value of item; may be a scalar or array
631 
632  Notes
633  -----
634  If `value` is an `lsst.daf.base.PropertySet` items are added
635  using dotted names (e.g. if name="a" and value contains
636  an item "b" which is another PropertySet and contains an
637  item "c" which is numeric or string, then the value of "c"
638  is added as "a.b.c", appended to the existing values of
639  "a.b.c" if any (in which case the types must be compatible).
640 
641  Raises
642  ------
643  lsst::pex::exceptions::TypeError
644  If the type of `value` is incompatible with the existing value
645  of the item.
646  """
647  args = []
648  if comment is not None:
649  args.append(comment)
650  return _propertyContainerAdd(self, name, value, self._typeMenu, *args)
651 
652  def setComment(self, name, comment):
653  """Set the comment for an existing entry.
654 
655  Parameters
656  ----------
657  name : `str`
658  Name of the key to receive updated comment.
659  comment : `comment`
660  New comment string.
661  """
662  # The only way to do this is to replace the existing entry with
663  # one that has the new comment
664  containerType = _propertyContainerElementTypeName(self, name)
665  if self.isArray(name):
666  value = self.getArray(name)
667  else:
668  value = self.getScalar(name)
669  getattr(self, f"set{containerType}")(name, value, comment)
670 
671  def toList(self):
672  """Return a list of tuples of name, value, comment for each property
673  in the order that they were inserted.
674 
675  Returns
676  -------
677  ret : `list` of `tuple`
678  Tuples of name, value, comment for each property in the order
679  in which they were inserted.
680  """
681  orderedNames = self.getOrderedNames()
682  ret = []
683  for name in orderedNames:
684  if self.isArray(name):
685  values = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
686  for v in values:
687  ret.append((name, v, self.getComment(name)))
688  else:
689  ret.append((name, _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO),
690  self.getComment(name)))
691  return ret
692 
693  def toOrderedDict(self):
694  """Return an ordered dictionary with all properties in the order that
695  they were inserted.
696 
697  Returns
698  -------
699  d : `dict`
700  Ordered dictionary with all properties in the order that they
701  were inserted. Comments are not included.
702 
703  Notes
704  -----
705  As of Python 3.6 dicts retain their insertion order.
706  """
707  d = {}
708  for name in self:
709  d[name] = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
710  return d
711 
712  # For PropertyList the two are equivalent
713  toDict = toOrderedDict
714 
715  def __eq__(self, other):
716  # super() doesn't seem to work properly in @continueClass;
717  # note that super with arguments seems to work at first, but actually
718  # doesn't either.
719  if not PropertySet.__eq__(self, other):
720  return False
721 
722  for name in self:
723  if self.getComment(name) != other.getComment(name):
724  return False
725 
726  return True
727 
728  def __copy__(self):
729  # Copy without having to go through pickle state
730  pl = PropertyList()
731  for itemName in self:
732  pl.copy(itemName, self, itemName)
733  return pl
734 
735  def __deepcopy__(self, memo):
736  result = self.deepCopy()
737  memo[id(self)] = result
738  return result
739 
740  def __iter__(self):
741  for n in self.getOrderedNames():
742  yield n
743 
744  def __setitem__(self, name, value):
745  if name.endswith(self.COMMENTSUFFIX):
746  name = name[:-len(self.COMMENTSUFFIX)]
747  self.setComment(name, value)
748  return
749  if isinstance(value, Mapping):
750  # Create a property set instead
751  ps = PropertySet()
752  for k, v in value.items():
753  ps[k] = v
754  value = ps
755  self.set(name, value)
756 
757  def __reduce__(self):
758  # It would be a bit simpler to use __setstate__ and __getstate__.
759  # However, implementing __setstate__ in Python causes segfaults
760  # because pickle creates a new instance by calling
761  # object.__new__(PropertyList, *args) which bypasses
762  # the pybind11 memory allocation step.
763  return (_makePropertyList, (getPropertyListState(self),))