lsst.daf.base  16.0-4-g50d071e+5
propertyContainerContinued.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <http://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 
25 __all__ = ["getPropertySetState", "getPropertyListState", "setPropertySetState", "setPropertyListState"]
26 
27 import enum
28 import numbers
29 import warnings
30 
31 from lsst.utils import continueClass
32 
33 from .propertySet import PropertySet
34 from .propertyList import PropertyList
36 from ..dateTime import DateTime
37 
38 
39 def getPropertySetState(container):
40  """Get the state of a PropertySet in a form that can be pickled.
41 
42  Parameters
43  ----------
44  container : `PropertySet`\
45  The property container.
46 
47  Returns
48  -------
49  state : `list`
50  The state, as a list of tuples, each of which contains
51  the following 3 items:
52  - name (a `str`): the name of the item
53  - elementTypeName (a `str`): the suffix of a ``setX`` method name
54  which is appropriate for the data type. For example integer
55  data has ``elementTypeName="Int"` which corresponds to
56  the ``setInt`` method.
57  - value: the data for the item, in a form compatible
58  with the set method named by ``elementTypeName``
59  """
60  return [(name, _propertyContainerElementTypeName(container, name),
61  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO))
62  for name in container.paramNames(False)]
63 
64 
65 def getPropertyListState(container):
66  """Get the state of a PropertyList in a form that can be pickled.
67 
68  Parameters
69  ----------
70  container : `PropertyList`
71  The property container.
72 
73  Returns
74  -------
75  state : `list`
76  The state, as a list of tuples, each of which contains
77  the following 4 items:
78  - name (a `str`): the name of the item
79  - elementTypeName (a `str`): the suffix of a ``setX`` method name
80  which is appropriate for the data type. For example integer
81  data has ``elementTypeName="Int"` which corresponds to
82  the ``setInt`` method.
83  - value: the data for the item, in a form compatible
84  with the set method named by ``elementTypeName``
85  - comment (a `str`): the comment. This item is only present
86  if ``container`` is a PropertyList.
87  """
88  return [(name, _propertyContainerElementTypeName(container, name),
89  _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO),
90  container.getComment(name))
91  for name in container.getOrderedNames()]
92 
93 
94 def setPropertySetState(container, state):
95  """Restore the state of a PropertySet, in place.
96 
97  Parameters
98  ----------
99  container : `PropertySet`
100  The property container whose state is to be restored.
101  It should be empty to start with and is updated in place.
102  state : `list`
103  The state, as returned by ``getPropertySetState``
104  """
105  for name, elemType, value in state:
106  getattr(container, "set" + elemType)(name, value)
107 
108 
109 def setPropertyListState(container, state):
110  """Restore the state of a PropertyList, in place.
111 
112  Parameters
113  ----------
114  container : `PropertyList`
115  The property container whose state is to be restored.
116  It should be empty to start with and is updated in place.
117  state : `list`
118  The state, as returned by ``getPropertyListState``
119  """
120  for name, elemType, value, comment in state:
121  getattr(container, "set" + elemType)(name, value, comment)
122 
123 
124 class ReturnStyle(enum.Enum):
125  ARRAY = enum.auto()
126  SCALAR = enum.auto()
127  AUTO = enum.auto()
128 
129 
130 def _propertyContainerElementTypeName(container, name):
131  """Return name of the type of a particular element"""
132  t = container.typeOf(name)
133  for checkType in ("Bool", "Short", "Int", "Long", "LongLong", "Float", "Double", "String", "DateTime"):
134  if t == getattr(container, "TYPE_" + checkType):
135  return checkType
136  return None
137 
138 
139 def _propertyContainerGet(container, name, returnStyle):
140  """Get a value of unknown type as a scalar or array
141 
142  Parameters
143  ----------
144  container : ``lsst.daf.base.PropertySet`` or ``lsst.daf.base.PropertyList``
145  Container from which to get the value
146  name : ``str``
147  Name of item
148  returnStyle : ``ReturnStyle``
149  Control whether numeric or string data is returned as an array
150  or scalar (the other types, ``PropertyList``, ``PropertySet``
151  and ``PersistablePtr``, are always returned as a scalar):
152  - ReturnStyle.ARRAY: return numeric or string data types
153  as an array of values.
154  - ReturnStyle.SCALAR: return numeric or string data types
155  as a single value; if the item has multiple values then
156  return the last value.
157  - ReturnStyle.AUTO: (deprecated) return numeric or string data
158  as a scalar if there is just one item, or as an array
159  otherwise.
160  """
161  if not container.exists(name):
162  raise lsst.pex.exceptions.NotFoundError(name + " not found")
163  if returnStyle not in ReturnStyle:
164  raise ValueError("returnStyle {} must be a ReturnStyle".format(returnStyle))
165 
166  elemType = _propertyContainerElementTypeName(container, name)
167  if elemType:
168  value = getattr(container, "getArray" + elemType)(name)
169  if returnStyle == ReturnStyle.ARRAY or (returnStyle == ReturnStyle.AUTO and len(value) > 1):
170  return value
171  return value[-1]
172 
173  try:
174  return container.getAsPropertyListPtr(name)
175  except Exception:
176  pass
177  if container.typeOf(name) == container.TYPE_PropertySet:
178  return container.getAsPropertySetPtr(name)
179  try:
180  return container.getAsPersistablePtr(name)
181  except Exception:
182  pass
183  raise lsst.pex.exceptions.TypeError('Unknown PropertySet value type for ' + name)
184 
185 
186 def _guessIntegerType(container, name, value):
187  """Given an existing container and name, determine the type
188  that should be used for the supplied value. The supplied value
189  is assumed to be a scalar.
190 
191  On Python 3 all ints are LongLong but we need to be able to store them
192  in Int containers if that is what is being used (testing for truncation).
193  Int is assumed to mean 32bit integer (2147483647 to -2147483648).
194 
195  If there is no pre-existing value we have to decide what to do. For now
196  we pick Int if the value is less than maxsize.
197 
198  Returns None if the value supplied is a bool or not an integral value.
199  """
200  useType = None
201  maxInt = 2147483647
202  minInt = -2147483648
203 
204  # We do not want to convert bool to int so let the system work that
205  # out itself
206  if isinstance(value, bool):
207  return useType
208 
209  if isinstance(value, numbers.Integral):
210  try:
211  containerType = _propertyContainerElementTypeName(container, name)
213  # nothing in the container so choose based on size. Safe option is to
214  # always use LongLong
215  if value <= maxInt and value >= minInt:
216  useType = "Int"
217  else:
218  useType = "LongLong"
219  else:
220  if containerType == "Int":
221  # Always use an Int even if we know it won't fit. The later
222  # code will trigger OverflowError if appropriate. Setting the
223  # type to LongLong here will trigger a TypeError instead so it's
224  # best to trigger a predictable OverflowError.
225  useType = "Int"
226  elif containerType == "LongLong":
227  useType = "LongLong"
228  return useType
229 
230 
231 def _propertyContainerSet(container, name, value, typeMenu, *args):
232  """Set a single Python value of unknown type"""
233  if hasattr(value, "__iter__") and not isinstance(value, str):
234  exemplar = value[0]
235  else:
236  exemplar = value
237 
238  t = type(exemplar)
239  setType = _guessIntegerType(container, name, exemplar)
240 
241  if setType is not None or t in typeMenu:
242  if setType is None:
243  setType = typeMenu[t]
244  return getattr(container, "set" + setType)(name, value, *args)
245  # Allow for subclasses
246  for checkType in typeMenu:
247  if isinstance(exemplar, checkType):
248  return getattr(container, "set" + typeMenu[checkType])(name, value, *args)
249  raise lsst.pex.exceptions.TypeError("Unknown value type for %s: %s" % (name, t))
250 
251 
252 def _propertyContainerAdd(container, name, value, typeMenu, *args):
253  """Add a single Python value of unknown type"""
254  if hasattr(value, "__iter__"):
255  exemplar = value[0]
256  else:
257  exemplar = value
258 
259  t = type(exemplar)
260  addType = _guessIntegerType(container, name, exemplar)
261 
262  if addType is not None or t in typeMenu:
263  if addType is None:
264  addType = typeMenu[t]
265  return getattr(container, "add" + addType)(name, value, *args)
266  # Allow for subclasses
267  for checkType in typeMenu:
268  if isinstance(exemplar, checkType):
269  return getattr(container, "add" + typeMenu[checkType])(name, value, *args)
270  raise lsst.pex.exceptions.TypeError("Unknown value type for %s: %s" % (name, t))
271 
272 
273 def _makePropertySet(state):
274  """Make a ``PropertySet`` from the state returned by ``getstate``
275 
276  Parameters
277  ----------
278  state : `list`
279  The data returned by ``getstate``.
280  """
281  ps = PropertySet()
282  setPropertySetState(ps, state)
283  return ps
284 
285 
286 def _makePropertyList(state):
287  """Make a ``PropertyList`` from the state returned by ``getstate``
288 
289  Parameters
290  ----------
291  state : `list`
292  The data returned by ``getstate``.
293  """
294  pl = PropertyList()
295  setPropertyListState(pl, state)
296  return pl
297 
298 
299 @continueClass
301  # Mapping of type to method names;
302  # int types are omitted due to use of _guessIntegerType
303  _typeMenu = {bool: "Bool",
304  float: "Double",
305  str: "String",
306  DateTime: "DateTime",
307  PropertySet: "PropertySet",
308  PropertyList: "PropertySet",
309  }
310 
311  # Map unicode to String, but this only works on Python 2
312  # so catch the error and do nothing on Python 3.
313  try:
314  _typeMenu[unicode] = "String" # noqa F821
315  except Exception:
316  pass
317 
318  def get(self, name):
319  """Return an item as a scalar or array
320 
321  Return an array if the item is of numeric or string type and has
322  more than one value, otherwise return a scalar.
323 
324  .. deprecated:: 20180-06
325  `get` is superseded by `getArray` or `getScalar`
326 
327  Parameters
328  ----------
329  name : ``str``
330  Name of item
331 
332  Raises
333  ------
334  lsst.pex.exceptions.NotFoundError
335  If the item does not exist.
336  """
337  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
338  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
339 
340  def getArray(self, name):
341  """Return an item as an array if the item is numeric or string
342 
343  If the item is a ``PropertySet``, ``PropertyList`` or
344  ``lsst.daf.base.PersistablePtr`` then return the item as a scalar.
345 
346  Parameters
347  ----------
348  name : ``str``
349  Name of item
350 
351  Raises
352  ------
353  lsst.pex.exceptions.NotFoundError
354  If the item does not exist.
355  """
356  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
357 
358  def getScalar(self, name):
359  """Return an item as a scalar
360 
361  If the item has more than one value then the last value is returned
362 
363  Parameters
364  ----------
365  name : ``str``
366  Name of item
367 
368  Raises
369  ------
370  lsst.pex.exceptions.NotFoundError
371  If the item does not exist.
372  """
373  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
374 
375  def set(self, name, value):
376  """Set the value of an item
377 
378  If the item already exists it is silently replaced; the types
379  need not match.
380 
381  Parameters
382  ----------
383  name : ``str``
384  Name of item
385  value : any supported type
386  Value of item; may be a scalar or array
387  """
388  return _propertyContainerSet(self, name, value, self._typeMenu)
389 
390  def add(self, name, value):
391  """Append one or more values to a given item, which need not exist
392 
393  If the item exists then the new value(s) are appended;
394  otherwise it is like calling `set`
395 
396  Parameters
397  ----------
398  name : ``str``
399  Name of item
400  value : any supported type
401  Value of item; may be a scalar or array
402 
403  Notes
404  -----
405  If `value` is an ``lsst.daf.base.PropertySet`` or
406  ``lsst.daf.base.PropertyList`` then `value` replaces
407  the existing value. Also the item is added as a live
408  reference, so updating `value` will update this container
409  and vice-versa.
410 
411  Raises
412  ------
413  lsst::pex::exceptions::TypeError
414  If the type of `value` is incompatible with the existing value
415  of the item.
416  """
417  return _propertyContainerAdd(self, name, value, self._typeMenu)
418 
419  def toDict(self):
420  """Returns a (possibly nested) dictionary with all properties.
421  """
422 
423  d = {}
424  for name in self.names():
425  v = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
426 
427  if isinstance(v, PropertySet):
428  d[name] = PropertySet.toDict(v)
429  else:
430  d[name] = v
431  return d
432 
433  def __reduce__(self):
434  # It would be a bit simpler to use __setstate__ and __getstate__.
435  # However, implementing __setstate__ in Python causes segfaults
436  # because pickle creates a new instance by calling
437  # object.__new__(PropertyList, *args) which bypasses
438  # the pybind11 memory allocation step.
439  return (_makePropertySet, (getPropertySetState(self),))
440 
441 
442 @continueClass
444  # Mapping of type to method names
445  _typeMenu = {bool: "Bool",
446  int: "Int",
447  float: "Double",
448  str: "String",
449  DateTime: "DateTime",
450  PropertySet: "PropertySet",
451  PropertyList: "PropertySet",
452  }
453 
454  # Map unicode to String, but this only works on Python 2
455  # so catch the error and do nothing on Python 3.
456  try:
457  _typeMenu[unicode] = "String" # noqa F821
458  except Exception:
459  pass
460 
461  def get(self, name):
462  """Return an item as a scalar or array
463 
464  Return an array if the item has more than one value,
465  otherwise return a scalar.
466 
467  .. deprecated:: 20180-06
468  `get` is superseded by `getArray` or `getScalar`
469 
470  Parameters
471  ----------
472  name : ``str``
473  Name of item
474 
475  Raises
476  ------
477  lsst.pex.exceptions.NotFoundError
478  If the item does not exist.
479  """
480  warnings.warn("Use getArray or getScalar instead", DeprecationWarning, stacklevel=2)
481  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
482 
483  def getArray(self, name):
484  """Return an item as an array
485 
486  Parameters
487  ----------
488  name : ``str``
489  Name of item
490 
491  Raises
492  ------
493  lsst.pex.exceptions.NotFoundError
494  If the item does not exist.
495  """
496  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)
497 
498  def getScalar(self, name):
499  """Return an item as a scalar
500 
501  If the item has more than one value then the last value is returned
502 
503  Parameters
504  ----------
505  name : ``str``
506  Name of item
507 
508  Raises
509  ------
510  lsst.pex.exceptions.NotFoundError
511  If the item does not exist.
512  """
513  return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)
514 
515  def set(self, name, value, comment=None):
516  """Set the value of an item
517 
518  If the item already exists it is silently replaced; the types
519  need not match.
520 
521  Parameters
522  ----------
523  name : ``str``
524  Name of item
525  value : any supported type
526  Value of item; may be a scalar or array
527  """
528  args = []
529  if comment is not None:
530  args.append(comment)
531  return _propertyContainerSet(self, name, value, self._typeMenu, *args)
532 
533  def add(self, name, value, comment=None):
534  """Append one or more values to a given item, which need not exist
535 
536  If the item exists then the new value(s) are appended;
537  otherwise it is like calling `set`
538 
539  Parameters
540  ----------
541  name : ``str``
542  Name of item
543  value : any supported type
544  Value of item; may be a scalar or array
545 
546  Notes
547  -----
548  If `value` is an ``lsst.daf.base.PropertySet`` items are added
549  using dotted names (e.g. if name="a" and value contains
550  an item "b" which is another PropertySet and contains an
551  item "c" which is numeric or string, then the value of "c"
552  is added as "a.b.c", appended to the existing values of
553  "a.b.c" if any (in which case the types must be compatible).
554 
555  Raises
556  ------
557  lsst::pex::exceptions::TypeError
558  If the type of `value` is incompatible with the existing value
559  of the item.
560  """
561  args = []
562  if comment is not None:
563  args.append(comment)
564  return _propertyContainerAdd(self, name, value, self._typeMenu, *args)
565 
566  def toList(self):
567  orderedNames = self.getOrderedNames()
568  ret = []
569  for name in orderedNames:
570  if self.isArray(name):
571  values = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
572  for v in values:
573  ret.append((name, v, self.getComment(name)))
574  else:
575  ret.append((name, _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO),
576  self.getComment(name)))
577  return ret
578 
579  def toOrderedDict(self):
580  """Return an ordered dictionary with all properties in the order that
581  they were inserted.
582  """
583  from collections import OrderedDict
584 
585  d = OrderedDict()
586  for name in self.getOrderedNames():
587  d[name] = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)
588  return d
589 
590  def __reduce__(self):
591  # It would be a bit simpler to use __setstate__ and __getstate__.
592  # However, implementing __setstate__ in Python causes segfaults
593  # because pickle creates a new instance by calling
594  # object.__new__(PropertyList, *args) which bypasses
595  # the pybind11 memory allocation step.
596  return (_makePropertyList, (getPropertyListState(self),))