lsst.daf.base  16.0-5-g4940a70
propertyContainerContinued.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 
25 __all__ = ["getPropertySetState", "getPropertyListState", "setPropertySetState", "setPropertyListState"]
26 
27 import enum
28 import numbers
29 import warnings
30 
31 from lsst.utils import continueClass
32 
33 from .propertySet import PropertySet
34 from .propertyList import PropertyList
36 from ..dateTime import DateTime
37 
38 
39 def getPropertySetState(container, asLists=False):
40  """Get the state of a PropertySet in a form that can be pickled.
41 
42  Parameters
43  ----------
44  container : `PropertySet`
45  The property container.
46  asLists : `bool`, optional
47  If False, the default, `tuple` will be used for the contents. If true
48  a `list` will be used.
49 
50  Returns
51  -------
52  state : `list` of `tuple` or `list` of `list`
53  The state, as a list of tuples (or lists), each of which contains
54  the following 3 items:
55  - name (a `str`): the name of the item
56  - elementTypeName (a `str`): the suffix of a ``setX`` method name
57  which is appropriate for the data type. For example integer
58  data has ``elementTypeName="Int"` which corresponds to
59  the ``setInt`` method.
60  - value: the data for the item, in a form compatible
61  with the set method named by ``elementTypeName``
62  """
63  sequence = list if asLists else tuple
64  return [sequence((name, _propertyContainerElementTypeName(container, name),
65  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO)))
66  for name in container.paramNames(False)]
67 
68 
69 def getPropertyListState(container, asLists=False):
70  """Get the state of a PropertyList in a form that can be pickled.
71 
72  Parameters
73  ----------
74  container : `PropertyList`
75  The property container.
76  asLists : `bool`, optional
77  If False, the default, `tuple` will be used for the contents. If true
78  a `list` will be used.
79 
80  Returns
81  -------
82  state : `list` of `tuple` or `list` of `list`
83  The state, as a list of tuples (or lists), each of which contains
84  the following 4 items:
85  - name (a `str`): the name of the item
86  - elementTypeName (a `str`): the suffix of a ``setX`` method name
87  which is appropriate for the data type. For example integer
88  data has ``elementTypeName="Int"` which corresponds to
89  the ``setInt`` method.
90  - value: the data for the item, in a form compatible
91  with the set method named by ``elementTypeName``
92  - comment (a `str`): the comment. This item is only present
93  if ``container`` is a PropertyList.
94  """
95  sequence = list if asLists else tuple
96  return [sequence((name, _propertyContainerElementTypeName(container, name),
97  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO),
98  container.getComment(name)))
99  for name in container.getOrderedNames()]
100 
101 
102 def setPropertySetState(container, state):
103  """Restore the state of a PropertySet, in place.
104 
105  Parameters
106  ----------
107  container : `PropertySet`
108  The property container whose state is to be restored.
109  It should be empty to start with and is updated in place.
110  state : `list`
111  The state, as returned by ``getPropertySetState``
112  """
113  for name, elemType, value in state:
114  getattr(container, "set" + elemType)(name, value)
115 
116 
117 def setPropertyListState(container, state):
118  """Restore the state of a PropertyList, in place.
119 
120  Parameters
121  ----------
122  container : `PropertyList`
123  The property container whose state is to be restored.
124  It should be empty to start with and is updated in place.
125  state : `list`
126  The state, as returned by ``getPropertyListState``
127  """
128  for name, elemType, value, comment in state:
129  getattr(container, "set" + elemType)(name, value, comment)
130 
131 
132 class ReturnStyle(enum.Enum):
133  ARRAY = enum.auto()
134  SCALAR = enum.auto()
135  AUTO = enum.auto()
136 
137 
138 def _propertyContainerElementTypeName(container, name):
139  """Return name of the type of a particular element"""
140  t = container.typeOf(name)
141  for checkType in ("Bool", "Short", "Int", "Long", "LongLong", "Float", "Double", "String", "DateTime"):
142  if t == getattr(container, "TYPE_" + checkType):
143  return checkType
144  return None
145 
146 
147 def _propertyContainerGet(container, name, returnStyle):
148  """Get a value of unknown type as a scalar or array
149 
150  Parameters
151  ----------
152  container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`
153  Container from which to get the value
154  name : `str`
155  Name of item
156  returnStyle : `ReturnStyle`
157  Control whether numeric or string data is returned as an array
158  or scalar (the other types, ``PropertyList``, ``PropertySet``
159  and ``PersistablePtr``, are always returned as a scalar):
160  - ReturnStyle.ARRAY: return numeric or string data types
161  as an array of values.
162  - ReturnStyle.SCALAR: return numeric or string data types
163  as a single value; if the item has multiple values then
164  return the last value.
165  - ReturnStyle.AUTO: (deprecated) return numeric or string data
166  as a scalar if there is just one item, or as an array
167  otherwise.
168  """
169  if not container.exists(name):
170  raise lsst.pex.exceptions.NotFoundError(name + " not found")
171  if returnStyle not in ReturnStyle:
172  raise ValueError("returnStyle {} must be a ReturnStyle".format(returnStyle))
173 
174  elemType = _propertyContainerElementTypeName(container, name)
175  if elemType:
176  value = getattr(container, "getArray" + elemType)(name)
177  if returnStyle == ReturnStyle.ARRAY or (returnStyle == ReturnStyle.AUTO and len(value) > 1):
178  return value
179  return value[-1]
180 
181  try:
182  return container.getAsPropertyListPtr(name)
183  except Exception:
184  pass
185  if container.typeOf(name) == container.TYPE_PropertySet:
186  return container.getAsPropertySetPtr(name)
187  try:
188  return container.getAsPersistablePtr(name)
189  except Exception:
190  pass
191  raise lsst.pex.exceptions.TypeError('Unknown PropertySet value type for ' + name)
192 
193 
194 def _guessIntegerType(container, name, value):
195  """Given an existing container and name, determine the type
196  that should be used for the supplied value. The supplied value
197  is assumed to be a scalar.
198 
199  On Python 3 all ints are LongLong but we need to be able to store them
200  in Int containers if that is what is being used (testing for truncation).
201  Int is assumed to mean 32bit integer (2147483647 to -2147483648).
202 
203  If there is no pre-existing value we have to decide what to do. For now
204  we pick Int if the value is less than maxsize.
205 
206  Returns None if the value supplied is a bool or not an integral value.
207  """
208  useType = None
209  maxInt = 2147483647
210  minInt = -2147483648
211 
212  # We do not want to convert bool to int so let the system work that
213  # out itself
214  if isinstance(value, bool):
215  return useType
216 
217  if isinstance(value, numbers.Integral):
218  try:
219  containerType = _propertyContainerElementTypeName(container, name)
221  # nothing in the container so choose based on size. Safe option is to
222  # always use LongLong
223  if value <= maxInt and value >= minInt:
224  useType = "Int"
225  else:
226  useType = "LongLong"
227  else:
228  if containerType == "Int":
229  # Always use an Int even if we know it won't fit. The later
230  # code will trigger OverflowError if appropriate. Setting the
231  # type to LongLong here will trigger a TypeError instead so it's
232  # best to trigger a predictable OverflowError.
233  useType = "Int"
234  elif containerType == "LongLong":
235  useType = "LongLong"
236  return useType
237 
238 
239 def _propertyContainerSet(container, name, value, typeMenu, *args):
240  """Set a single Python value of unknown type"""
241  if hasattr(value, "__iter__") and not isinstance(value, str):
242  exemplar = value[0]
243  else:
244  exemplar = value
245 
246  t = type(exemplar)
247  setType = _guessIntegerType(container, name, exemplar)
248 
249  if setType is not None or t in typeMenu:
250  if setType is None:
251  setType = typeMenu[t]
252  return getattr(container, "set" + setType)(name, value, *args)
253  # Allow for subclasses
254  for checkType in typeMenu:
255  if isinstance(exemplar, checkType):
256  return getattr(container, "set" + typeMenu[checkType])(name, value, *args)
257  raise lsst.pex.exceptions.TypeError("Unknown value type for %s: %s" % (name, t))
258 
259 
260 def _propertyContainerAdd(container, name, value, typeMenu, *args):
261  """Add a single Python value of unknown type"""
262  if hasattr(value, "__iter__"):
263  exemplar = value[0]
264  else:
265  exemplar = value
266 
267  t = type(exemplar)
268  addType = _guessIntegerType(container, name, exemplar)
269 
270  if addType is not None or t in typeMenu:
271  if addType is None:
272  addType = typeMenu[t]
273  return getattr(container, "add" + addType)(name, value, *args)
274  # Allow for subclasses
275  for checkType in typeMenu:
276  if isinstance(exemplar, checkType):
277  return getattr(container, "add" + typeMenu[checkType])(name, value, *args)
278  raise lsst.pex.exceptions.TypeError("Unknown value type for %s: %s" % (name, t))
279 
280 
281 def _makePropertySet(state):
282  """Make a `PropertySet` from the state returned by `getPropertySetState`
283 
284  Parameters
285  ----------
286  state : `list`
287  The data returned by `getPropertySetState`.
288  """
289  ps = PropertySet()
290  setPropertySetState(ps, state)
291  return ps
292 
293 
294 def _makePropertyList(state):
295  """Make a `PropertyList` from the state returned by
296  `getPropertyListState`
297 
298  Parameters
299  ----------
300  state : `list`
301  The data returned by `getPropertySetState`.
302  """
303  pl = PropertyList()
304  setPropertyListState(pl, state)
305  return pl
306 
307 
308 @continueClass
310  # Mapping of type to method names;
311  # int types are omitted due to use of _guessIntegerType
312  _typeMenu = {bool: "Bool",
313  float: "Double",
314  str: "String",
315  DateTime: "DateTime",
316  PropertySet: "PropertySet",
317  PropertyList: "PropertySet",
318  }
319 
320  # Map unicode to String, but this only works on Python 2
321  # so catch the error and do nothing on Python 3.
322  try:
323  _typeMenu[unicode] = "String" # noqa F821
324  except Exception:
325  pass
326 
327  def get(self, name):
328  """Return an item as a scalar or array
329 
330  Return an array if the item is of numeric or string type and has
331  more than one value, otherwise return a scalar.
332 
333  .. deprecated:: 20180-06
334  `get` is superseded by `getArray` or `getScalar`
335 
336  Parameters
337  ----------
338  name : ``str``
339  Name of item
340 
341  Raises
342  ------
343  lsst.pex.exceptions.NotFoundError
344  If the item does not exist.
345  """
346  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
347  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
348 
349  def getArray(self, name):
350  """Return an item as an array if the item is numeric or string
351 
352  If the item is a `PropertySet`, `PropertyList` or
353  ``lsst.daf.base.PersistablePtr`` then return the item as a scalar.
354 
355  Parameters
356  ----------
357  name : `str`
358  Name of item
359 
360  Raises
361  ------
362  lsst.pex.exceptions.NotFoundError
363  If the item does not exist.
364  """
365  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
366 
367  def getScalar(self, name):
368  """Return an item as a scalar
369 
370  If the item has more than one value then the last value is returned
371 
372  Parameters
373  ----------
374  name : `str`
375  Name of item
376 
377  Raises
378  ------
379  lsst.pex.exceptions.NotFoundError
380  If the item does not exist.
381  """
382  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
383 
384  def set(self, name, value):
385  """Set the value of an item
386 
387  If the item already exists it is silently replaced; the types
388  need not match.
389 
390  Parameters
391  ----------
392  name : `str`
393  Name of item
394  value : any supported type
395  Value of item; may be a scalar or array
396  """
397  return _propertyContainerSet(self, name, value, self._typeMenu)
398 
399  def add(self, name, value):
400  """Append one or more values to a given item, which need not exist
401 
402  If the item exists then the new value(s) are appended;
403  otherwise it is like calling `set`
404 
405  Parameters
406  ----------
407  name : `str`
408  Name of item
409  value : any supported type
410  Value of item; may be a scalar or array
411 
412  Notes
413  -----
414  If ``value`` is an `lsst.daf.base.PropertySet` or
415  `lsst.daf.base.PropertyList` then ``value`` replaces
416  the existing value. Also the item is added as a live
417  reference, so updating ``value`` will update this container
418  and vice-versa.
419 
420  Raises
421  ------
422  lsst::pex::exceptions::TypeError
423  If the type of `value` is incompatible with the existing value
424  of the item.
425  """
426  return _propertyContainerAdd(self, name, value, self._typeMenu)
427 
428  def toDict(self):
429  """Returns a (possibly nested) dictionary with all properties.
430 
431  Returns
432  -------
433  d : `dict`
434  Dictionary with all names and values (no comments).
435  """
436 
437  d = {}
438  for name in self.names():
439  v = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
440 
441  if isinstance(v, PropertySet):
442  d[name] = PropertySet.toDict(v)
443  else:
444  d[name] = v
445  return d
446 
447  def __reduce__(self):
448  # It would be a bit simpler to use __setstate__ and __getstate__.
449  # However, implementing __setstate__ in Python causes segfaults
450  # because pickle creates a new instance by calling
451  # object.__new__(PropertyList, *args) which bypasses
452  # the pybind11 memory allocation step.
453  return (_makePropertySet, (getPropertySetState(self),))
454 
455 
456 @continueClass
458  # Mapping of type to method names
459  _typeMenu = {bool: "Bool",
460  int: "Int",
461  float: "Double",
462  str: "String",
463  DateTime: "DateTime",
464  PropertySet: "PropertySet",
465  PropertyList: "PropertySet",
466  }
467 
468  # Map unicode to String, but this only works on Python 2
469  # so catch the error and do nothing on Python 3.
470  try:
471  _typeMenu[unicode] = "String" # noqa F821
472  except Exception:
473  pass
474 
475  def get(self, name):
476  """Return an item as a scalar or array
477 
478  Return an array if the item has more than one value,
479  otherwise return a scalar.
480 
481  .. deprecated:: 20180-06
482  `get` is superseded by `getArray` or `getScalar`
483 
484  Parameters
485  ----------
486  name : `str`
487  Name of item
488 
489  Raises
490  ------
491  lsst.pex.exceptions.NotFoundError
492  If the item does not exist.
493  """
494  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
495  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
496 
497  def getArray(self, name):
498  """Return an item as an array
499 
500  Parameters
501  ----------
502  name : `str`
503  Name of item
504 
505  Raises
506  ------
507  lsst.pex.exceptions.NotFoundError
508  If the item does not exist.
509  """
510  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
511 
512  def getScalar(self, name):
513  """Return an item as a scalar
514 
515  If the item has more than one value then the last value is returned
516 
517  Parameters
518  ----------
519  name : `str`
520  Name of item
521 
522  Raises
523  ------
524  lsst.pex.exceptions.NotFoundError
525  If the item does not exist.
526  """
527  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
528 
529  def set(self, name, value, comment=None):
530  """Set the value of an item
531 
532  If the item already exists it is silently replaced; the types
533  need not match.
534 
535  Parameters
536  ----------
537  name : `str`
538  Name of item
539  value : any supported type
540  Value of item; may be a scalar or array
541  """
542  args = []
543  if comment is not None:
544  args.append(comment)
545  return _propertyContainerSet(self, name, value, self._typeMenu, *args)
546 
547  def add(self, name, value, comment=None):
548  """Append one or more values to a given item, which need not exist
549 
550  If the item exists then the new value(s) are appended;
551  otherwise it is like calling `set`
552 
553  Parameters
554  ----------
555  name : `str`
556  Name of item
557  value : any supported type
558  Value of item; may be a scalar or array
559 
560  Notes
561  -----
562  If `value` is an `lsst.daf.base.PropertySet` items are added
563  using dotted names (e.g. if name="a" and value contains
564  an item "b" which is another PropertySet and contains an
565  item "c" which is numeric or string, then the value of "c"
566  is added as "a.b.c", appended to the existing values of
567  "a.b.c" if any (in which case the types must be compatible).
568 
569  Raises
570  ------
571  lsst::pex::exceptions::TypeError
572  If the type of `value` is incompatible with the existing value
573  of the item.
574  """
575  args = []
576  if comment is not None:
577  args.append(comment)
578  return _propertyContainerAdd(self, name, value, self._typeMenu, *args)
579 
580  def toList(self):
581  """Return a list of tuples of name, value, comment for each property
582  in the order that they were inserted.
583 
584  Returns
585  -------
586  ret : `list` of `tuple`
587  Tuples of name, value, comment for each property in the order
588  in which they were inserted.
589  """
590  orderedNames = self.getOrderedNames()
591  ret = []
592  for name in orderedNames:
593  if self.isArray(name):
594  values = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
595  for v in values:
596  ret.append((name, v, self.getComment(name)))
597  else:
598  ret.append((name, _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO),
599  self.getComment(name)))
600  return ret
601 
602  def toOrderedDict(self):
603  """Return an ordered dictionary with all properties in the order that
604  they were inserted.
605 
606  Returns
607  -------
608  d : `~collections.OrderedDict`
609  Ordered dictionary with all properties in the order that they
610  were inserted. Comments are not included.
611  """
612  from collections import OrderedDict
613 
614  d = OrderedDict()
615  for name in self.getOrderedNames():
616  d[name] = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
617  return d
618 
619  def __reduce__(self):
620  # It would be a bit simpler to use __setstate__ and __getstate__.
621  # However, implementing __setstate__ in Python causes segfaults
622  # because pickle creates a new instance by calling
623  # object.__new__(PropertyList, *args) which bypasses
624  # the pybind11 memory allocation step.
625  return (_makePropertyList, (getPropertyListState(self),))