lsst.daf.base  16.0-7-gc370964+3
propertyContainerContinued.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 
25 __all__ = ["getPropertySetState", "getPropertyListState", "setPropertySetState", "setPropertyListState"]
26 
27 import enum
28 import numbers
29 import warnings
30 from collections.abc import Mapping, KeysView
31 
32 from lsst.utils import continueClass
33 
34 from .propertySet import PropertySet
35 from .propertyList import PropertyList
36 from ..dateTime import DateTime
37 
38 
39 def getPropertySetState(container, asLists=False, names=None):
40  """Get the state of a PropertySet in a form that can be pickled.
41 
42  Parameters
43  ----------
44  container : `PropertySet`
45  The property container.
46  asLists : `bool`, optional
47  If False, the default, `tuple` will be used for the contents. If true
48  a `list` will be used.
49  names : `list` or `tuple`
50  Override the default list of names with this subset.
51 
52  Returns
53  -------
54  state : `list` of `tuple` or `list` of `list`
55  The state, as a list of tuples (or lists), each of which contains
56  the following 3 items:
57  - name (a `str`): the name of the item
58  - elementTypeName (a `str`): the suffix of a ``setX`` method name
59  which is appropriate for the data type. For example integer
60  data has ``elementTypeName="Int"` which corresponds to
61  the ``setInt`` method.
62  - value: the data for the item, in a form compatible
63  with the set method named by ``elementTypeName``
64  """
65  if names is None:
66  # All top level names: this allows hierarchical PropertySet and
67  # PropertyList to be represented as their own entities. Without
68  # this a PropertyList inside a PropertySet loses all comments
69  # and becomes a PropertySet.
70  names = container.names(topLevelOnly=True)
71  sequence = list if asLists else tuple
72  return [sequence((name, _propertyContainerElementTypeName(container, name),
73  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO)))
74  for name in names]
75 
76 
77 def getPropertyListState(container, asLists=False):
78  """Get the state of a PropertyList in a form that can be pickled.
79 
80  Parameters
81  ----------
82  container : `PropertyList`
83  The property container.
84  asLists : `bool`, optional
85  If False, the default, `tuple` will be used for the contents. If true
86  a `list` will be used.
87 
88  Returns
89  -------
90  state : `list` of `tuple` or `list` of `list`
91  The state, as a list of tuples (or lists), each of which contains
92  the following 4 items:
93  - name (a `str`): the name of the item
94  - elementTypeName (a `str`): the suffix of a ``setX`` method name
95  which is appropriate for the data type. For example integer
96  data has ``elementTypeName="Int"` which corresponds to
97  the ``setInt`` method.
98  - value: the data for the item, in a form compatible
99  with the set method named by ``elementTypeName``
100  - comment (a `str`): the comment. This item is only present
101  if ``container`` is a PropertyList.
102  """
103  sequence = list if asLists else tuple
104  return [sequence((name, _propertyContainerElementTypeName(container, name),
105  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO),
106  container.getComment(name)))
107  for name in container.getOrderedNames()]
108 
109 
110 def setPropertySetState(container, state):
111  """Restore the state of a PropertySet, in place.
112 
113  Parameters
114  ----------
115  container : `PropertySet`
116  The property container whose state is to be restored.
117  It should be empty to start with and is updated in place.
118  state : `list`
119  The state, as returned by `getPropertySetState`
120  """
121  for name, elemType, value in state:
122  if elemType is not None:
123  getattr(container, "set" + elemType)(name, value)
124  else:
125  raise ValueError(f"Unrecognized values for state restoration: ({name}, {elemType}, {value})")
126 
127 
128 def setPropertyListState(container, state):
129  """Restore the state of a PropertyList, in place.
130 
131  Parameters
132  ----------
133  container : `PropertyList`
134  The property container whose state is to be restored.
135  It should be empty to start with and is updated in place.
136  state : `list`
137  The state, as returned by ``getPropertyListState``
138  """
139  for name, elemType, value, comment in state:
140  getattr(container, "set" + elemType)(name, value, comment)
141 
142 
143 class ReturnStyle(enum.Enum):
144  ARRAY = enum.auto()
145  SCALAR = enum.auto()
146  AUTO = enum.auto()
147 
148 
149 def _propertyContainerElementTypeName(container, name):
150  """Return name of the type of a particular element"""
151  try:
152  t = container.typeOf(name)
153  except LookupError:
154  # KeyError is more commonly expected when asking for an element
155  # from a mapping.
156  raise KeyError
157  for checkType in ("Bool", "Short", "Int", "Long", "LongLong", "Float", "Double", "String", "DateTime",
158  "PropertySet"):
159  if t == getattr(container, "TYPE_" + checkType):
160  return checkType
161  return None
162 
163 
164 def _propertyContainerGet(container, name, returnStyle):
165  """Get a value of unknown type as a scalar or array
166 
167  Parameters
168  ----------
169  container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`
170  Container from which to get the value
171  name : `str`
172  Name of item
173  returnStyle : `ReturnStyle`
174  Control whether numeric or string data is returned as an array
175  or scalar (the other types, ``PropertyList``, ``PropertySet``
176  and ``PersistablePtr``, are always returned as a scalar):
177  - ReturnStyle.ARRAY: return numeric or string data types
178  as an array of values.
179  - ReturnStyle.SCALAR: return numeric or string data types
180  as a single value; if the item has multiple values then
181  return the last value.
182  - ReturnStyle.AUTO: (deprecated) return numeric or string data
183  as a scalar if there is just one item, or as an array
184  otherwise.
185 
186  Raises
187  ------
188  KeyError
189  The specified key does not exist in the container.
190  TypeError
191  The value retrieved is of an unexpected type.
192  ValueError
193  The value for ``returnStyle`` is not correct.
194  """
195  if not container.exists(name):
196  raise KeyError(name + " not found")
197  if returnStyle not in ReturnStyle:
198  raise ValueError("returnStyle {} must be a ReturnStyle".format(returnStyle))
199 
200  elemType = _propertyContainerElementTypeName(container, name)
201  if elemType and elemType != "PropertySet":
202  value = getattr(container, "getArray" + elemType)(name)
203  if returnStyle == ReturnStyle.ARRAY or (returnStyle == ReturnStyle.AUTO and len(value) > 1):
204  return value
205  return value[-1]
206 
207  if container.isPropertySetPtr(name):
208  try:
209  return container.getAsPropertyListPtr(name)
210  except Exception:
211  return container.getAsPropertySetPtr(name)
212  try:
213  return container.getAsPersistablePtr(name)
214  except Exception:
215  pass
216  raise TypeError('Unknown PropertySet value type for ' + name)
217 
218 
219 def _guessIntegerType(container, name, value):
220  """Given an existing container and name, determine the type
221  that should be used for the supplied value. The supplied value
222  is assumed to be a scalar.
223 
224  On Python 3 all ints are LongLong but we need to be able to store them
225  in Int containers if that is what is being used (testing for truncation).
226  Int is assumed to mean 32bit integer (2147483647 to -2147483648).
227 
228  If there is no pre-existing value we have to decide what to do. For now
229  we pick Int if the value is less than maxsize.
230 
231  Returns None if the value supplied is a bool or not an integral value.
232  """
233  useType = None
234  maxInt = 2147483647
235  minInt = -2147483648
236 
237  # We do not want to convert bool to int so let the system work that
238  # out itself
239  if isinstance(value, bool):
240  return useType
241 
242  if isinstance(value, numbers.Integral):
243  try:
244  containerType = _propertyContainerElementTypeName(container, name)
245  except LookupError:
246  # nothing in the container so choose based on size. Safe option is to
247  # always use LongLong
248  if value <= maxInt and value >= minInt:
249  useType = "Int"
250  else:
251  useType = "LongLong"
252  else:
253  if containerType == "Int":
254  # Always use an Int even if we know it won't fit. The later
255  # code will trigger OverflowError if appropriate. Setting the
256  # type to LongLong here will trigger a TypeError instead so it's
257  # best to trigger a predictable OverflowError.
258  useType = "Int"
259  elif containerType == "LongLong":
260  useType = "LongLong"
261  return useType
262 
263 
264 def _propertyContainerSet(container, name, value, typeMenu, *args):
265  """Set a single Python value of unknown type"""
266  if hasattr(value, "__iter__") and not isinstance(value, (str, PropertySet, PropertyList)):
267  exemplar = value[0]
268  else:
269  exemplar = value
270 
271  t = type(exemplar)
272  setType = _guessIntegerType(container, name, exemplar)
273 
274  if setType is not None or t in typeMenu:
275  if setType is None:
276  setType = typeMenu[t]
277  return getattr(container, "set" + setType)(name, value, *args)
278  # Allow for subclasses
279  for checkType in typeMenu:
280  if isinstance(exemplar, checkType):
281  return getattr(container, "set" + typeMenu[checkType])(name, value, *args)
282  raise TypeError("Unknown value type for %s: %s" % (name, t))
283 
284 
285 def _propertyContainerAdd(container, name, value, typeMenu, *args):
286  """Add a single Python value of unknown type"""
287  if hasattr(value, "__iter__"):
288  exemplar = value[0]
289  else:
290  exemplar = value
291 
292  t = type(exemplar)
293  addType = _guessIntegerType(container, name, exemplar)
294 
295  if addType is not None or t in typeMenu:
296  if addType is None:
297  addType = typeMenu[t]
298  return getattr(container, "add" + addType)(name, value, *args)
299  # Allow for subclasses
300  for checkType in typeMenu:
301  if isinstance(exemplar, checkType):
302  return getattr(container, "add" + typeMenu[checkType])(name, value, *args)
303  raise TypeError("Unknown value type for %s: %s" % (name, t))
304 
305 
306 def _makePropertySet(state):
307  """Make a `PropertySet` from the state returned by `getPropertySetState`
308 
309  Parameters
310  ----------
311  state : `list`
312  The data returned by `getPropertySetState`.
313  """
314  ps = PropertySet()
315  setPropertySetState(ps, state)
316  return ps
317 
318 
319 def _makePropertyList(state):
320  """Make a `PropertyList` from the state returned by
321  `getPropertyListState`
322 
323  Parameters
324  ----------
325  state : `list`
326  The data returned by `getPropertySetState`.
327  """
328  pl = PropertyList()
329  setPropertyListState(pl, state)
330  return pl
331 
332 
333 @continueClass
335  # Mapping of type to method names;
336  # int types are omitted due to use of _guessIntegerType
337  _typeMenu = {bool: "Bool",
338  float: "Double",
339  str: "String",
340  DateTime: "DateTime",
341  PropertySet: "PropertySet",
342  PropertyList: "PropertySet",
343  }
344 
345  def get(self, name):
346  """Return an item as a scalar or array
347 
348  Return an array if the item is of numeric or string type and has
349  more than one value, otherwise return a scalar.
350 
351  .. deprecated:: 20180-06
352  `get` is superseded by `getArray` or `getScalar`
353 
354  Parameters
355  ----------
356  name : ``str``
357  Name of item
358 
359  Raises
360  ------
361  KeyError
362  If the item does not exist.
363  """
364  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
365  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
366 
367  def getArray(self, name):
368  """Return an item as an array if the item is numeric or string
369 
370  If the item is a `PropertySet`, `PropertyList` or
371  `lsst.daf.base.PersistablePtr` then return the item as a scalar.
372 
373  Parameters
374  ----------
375  name : `str`
376  Name of item
377 
378  Raises
379  ------
380  KeyError
381  If the item does not exist.
382  """
383  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
384 
385  def getScalar(self, name):
386  """Return an item as a scalar
387 
388  If the item has more than one value then the last value is returned
389 
390  Parameters
391  ----------
392  name : `str`
393  Name of item
394 
395  Raises
396  ------
397  KeyError
398  If the item does not exist.
399  """
400  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
401 
402  def set(self, name, value):
403  """Set the value of an item
404 
405  If the item already exists it is silently replaced; the types
406  need not match.
407 
408  Parameters
409  ----------
410  name : `str`
411  Name of item
412  value : any supported type
413  Value of item; may be a scalar or array
414  """
415  return _propertyContainerSet(self, name, value, self._typeMenu)
416 
417  def add(self, name, value):
418  """Append one or more values to a given item, which need not exist
419 
420  If the item exists then the new value(s) are appended;
421  otherwise it is like calling `set`
422 
423  Parameters
424  ----------
425  name : `str`
426  Name of item
427  value : any supported type
428  Value of item; may be a scalar or array
429 
430  Notes
431  -----
432  If ``value`` is an `lsst.daf.base.PropertySet` or
433  `lsst.daf.base.PropertyList` then ``value`` replaces
434  the existing value. Also the item is added as a live
435  reference, so updating ``value`` will update this container
436  and vice-versa.
437 
438  Raises
439  ------
440  lsst::pex::exceptions::TypeError
441  If the type of `value` is incompatible with the existing value
442  of the item.
443  """
444  return _propertyContainerAdd(self, name, value, self._typeMenu)
445 
446  def toDict(self):
447  """Returns a (possibly nested) dictionary with all properties.
448 
449  Returns
450  -------
451  d : `dict`
452  Dictionary with all names and values (no comments).
453  """
454 
455  d = {}
456  for name in self.names():
457  v = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
458 
459  if isinstance(v, PropertySet):
460  d[name] = PropertySet.toDict(v)
461  else:
462  d[name] = v
463  return d
464 
465  def __eq__(self, other):
466  if type(self) != type(other):
467  return False
468 
469  if len(self) != len(other):
470  return False
471 
472  for name in self:
473  if _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO) != \
474  _propertyContainerGet(other, name, returnStyle=ReturnStyle.AUTO):
475  return False
476  if self.typeOf(name) != other.typeOf(name):
477  return False
478 
479  return True
480 
481  def __copy__(self):
482  # Provide a copy because by default __reduce__ is used and that
483  # will not shallow copy properly, we therefore use the same
484  # pickling code but restrict the names
485  state = getPropertySetState(self, names=self.names(topLevelOnly=True))
486  return _makePropertySet(state)
487 
488  def __contains__(self, name):
489  # Do not use exists() because that includes "."-delimited names
490  return name in self.names(topLevelOnly=True)
491 
492  def __setitem__(self, name, value):
493  if isinstance(value, Mapping):
494  # Create a property set instead
495  ps = PropertySet()
496  for k, v in value.items():
497  ps[k] = v
498  value = ps
499  self.set(name, value)
500 
501  def __delitem__(self, name):
502  if name in self:
503  self.remove(name)
504  else:
505  raise KeyError(f"{name} not present in dict")
506 
507  def __str__(self):
508  return self.toString()
509 
510  def __len__(self):
511  return self.nameCount(topLevelOnly=True)
512 
513  def __iter__(self):
514  for n in self.names(topLevelOnly=True):
515  yield n
516 
517  def keys(self):
518  return KeysView(self)
519 
520  def __reduce__(self):
521  # It would be a bit simpler to use __setstate__ and __getstate__.
522  # However, implementing __setstate__ in Python causes segfaults
523  # because pickle creates a new instance by calling
524  # object.__new__(PropertyList, *args) which bypasses
525  # the pybind11 memory allocation step.
526  return (_makePropertySet, (getPropertySetState(self),))
527 
528 
529 @continueClass
531  # Mapping of type to method names
532  _typeMenu = {bool: "Bool",
533  int: "Int",
534  float: "Double",
535  str: "String",
536  DateTime: "DateTime",
537  PropertySet: "PropertySet",
538  PropertyList: "PropertySet",
539  }
540 
541  COMMENTSUFFIX = "#COMMENT"
542 
543  def get(self, name):
544  """Return an item as a scalar or array
545 
546  Return an array if the item has more than one value,
547  otherwise return a scalar.
548 
549  .. deprecated:: 20180-06
550  `get` is superseded by `getArray` or `getScalar`
551 
552  Parameters
553  ----------
554  name : `str`
555  Name of item
556 
557  Raises
558  ------
559  KeyError
560  If the item does not exist.
561  """
562  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
563  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
564 
565  def getArray(self, name):
566  """Return an item as an array
567 
568  Parameters
569  ----------
570  name : `str`
571  Name of item
572 
573  Raises
574  ------
575  KeyError
576  If the item does not exist.
577  """
578  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
579 
580  def getScalar(self, name):
581  """Return an item as a scalar
582 
583  If the item has more than one value then the last value is returned
584 
585  Parameters
586  ----------
587  name : `str`
588  Name of item
589 
590  Raises
591  ------
592  KeyError
593  If the item does not exist.
594  """
595  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
596 
597  def set(self, name, value, comment=None):
598  """Set the value of an item
599 
600  If the item already exists it is silently replaced; the types
601  need not match.
602 
603  Parameters
604  ----------
605  name : `str`
606  Name of item
607  value : any supported type
608  Value of item; may be a scalar or array
609  """
610  args = []
611  if comment is not None:
612  args.append(comment)
613  return _propertyContainerSet(self, name, value, self._typeMenu, *args)
614 
615  def add(self, name, value, comment=None):
616  """Append one or more values to a given item, which need not exist
617 
618  If the item exists then the new value(s) are appended;
619  otherwise it is like calling `set`
620 
621  Parameters
622  ----------
623  name : `str`
624  Name of item
625  value : any supported type
626  Value of item; may be a scalar or array
627 
628  Notes
629  -----
630  If `value` is an `lsst.daf.base.PropertySet` items are added
631  using dotted names (e.g. if name="a" and value contains
632  an item "b" which is another PropertySet and contains an
633  item "c" which is numeric or string, then the value of "c"
634  is added as "a.b.c", appended to the existing values of
635  "a.b.c" if any (in which case the types must be compatible).
636 
637  Raises
638  ------
639  lsst::pex::exceptions::TypeError
640  If the type of `value` is incompatible with the existing value
641  of the item.
642  """
643  args = []
644  if comment is not None:
645  args.append(comment)
646  return _propertyContainerAdd(self, name, value, self._typeMenu, *args)
647 
648  def setComment(self, name, comment):
649  """Set the comment for an existing entry.
650 
651  Parameters
652  ----------
653  name : `str`
654  Name of the key to receive updated comment.
655  comment : `comment`
656  New comment string.
657  """
658  # The only way to do this is to replace the existing entry with
659  # one that has the new comment
660  containerType = _propertyContainerElementTypeName(self, name)
661  if self.isArray(name):
662  value = self.getArray(name)
663  else:
664  value = self.getScalar(name)
665  getattr(self, f"set{containerType}")(name, value, comment)
666 
667  def toList(self):
668  """Return a list of tuples of name, value, comment for each property
669  in the order that they were inserted.
670 
671  Returns
672  -------
673  ret : `list` of `tuple`
674  Tuples of name, value, comment for each property in the order
675  in which they were inserted.
676  """
677  orderedNames = self.getOrderedNames()
678  ret = []
679  for name in orderedNames:
680  if self.isArray(name):
681  values = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
682  for v in values:
683  ret.append((name, v, self.getComment(name)))
684  else:
685  ret.append((name, _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO),
686  self.getComment(name)))
687  return ret
688 
689  def toOrderedDict(self):
690  """Return an ordered dictionary with all properties in the order that
691  they were inserted.
692 
693  Returns
694  -------
695  d : `~collections.OrderedDict`
696  Ordered dictionary with all properties in the order that they
697  were inserted. Comments are not included.
698  """
699  from collections import OrderedDict
700 
701  d = OrderedDict()
702  for name in self.getOrderedNames():
703  d[name] = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
704  return d
705 
706  def __eq__(self, other):
707  if not super(PropertySet, self).__eq__(other):
708  return False
709 
710  for name in self:
711  if self.getComment(name) != other.getComment(name):
712  return False
713 
714  return True
715 
716  def __copy__(self):
717  # Provide a copy because by default __reduce__ is used and that
718  # will not shallow copy properly, we therefore use the same
719  # pickling code but restrict the names
720  state = getPropertyListState(self, names=self.getOrderedNames())
721  return _makePropertyList(state)
722 
723  def __iter__(self):
724  for n in self.getOrderedNames():
725  yield n
726 
727  def __setitem__(self, name, value):
728  if name.endswith(self.COMMENTSUFFIX):
729  name = name[:-len(self.COMMENTSUFFIX)]
730  self.setComment(name, value)
731  return
732  if isinstance(value, Mapping):
733  # Create a property set instead
734  ps = PropertySet()
735  for k, v in value.items():
736  ps[k] = v
737  value = ps
738  self.set(name, value)
739 
740  def __reduce__(self):
741  # It would be a bit simpler to use __setstate__ and __getstate__.
742  # However, implementing __setstate__ in Python causes segfaults
743  # because pickle creates a new instance by calling
744  # object.__new__(PropertyList, *args) which bypasses
745  # the pybind11 memory allocation step.
746  return (_makePropertyList, (getPropertyListState(self),))