23 from builtins
import str
24 from builtins
import object
25 from past.builtins
import long
26 from past.builtins
import basestring
27 from past.builtins
import unicode
38 from .comparison
import getComparisonName, compareScalars, compareConfigs
39 from future.utils
import with_metaclass
41 __all__ = (
"Config",
"Field",
"FieldValidationError")
44 def _joinNamePath(prefix=None, name=None, index=None):
46 Utility function for generating nested configuration names
48 if not prefix
and not name:
49 raise ValueError(
"Invalid name: cannot be None")
53 name = prefix +
"." + name
56 return "%s[%r]" % (name, index)
61 def _autocast(x, dtype):
63 If appropriate perform type casting of value x to type dtype,
64 otherwise return the original value x
66 if dtype == float
and isinstance(x, int):
68 if dtype == int
and isinstance(x, long):
70 if isinstance(x, str):
77 Utility function to generate a fully qualified type name.
79 This is used primarily in writing config files to be
80 executed later upon 'load'.
82 if hasattr(x,
'__module__')
and hasattr(x,
'__name__'):
86 if (sys.version_info.major <= 2
and xtype.__module__ ==
'__builtin__')
or xtype.__module__ ==
'builtins':
89 return "%s.%s" % (xtype.__module__, xtype.__name__)
93 """A metaclass for Config
95 Adds a dictionary containing all Field class attributes
96 as a class attribute called '_fields', and adds the name of each field as
97 an instance variable of the field itself (so you don't have to pass the
98 name of the field to the field constructor).
101 type.__init__(self, name, bases, dict_)
103 self.
_source = traceback.extract_stack(limit=2)[0]
105 def getFields(classtype):
107 bases = list(classtype.__bases__)
110 fields.update(getFields(b))
112 for k, v
in classtype.__dict__.items():
113 if isinstance(v, Field):
117 fields = getFields(self)
118 for k, v
in fields.items():
119 setattr(self, k, copy.deepcopy(v))
122 if isinstance(value, Field):
125 type.__setattr__(self, name, value)
130 Custom exception class which holds additional information useful to
131 debuggin Config errors:
132 - fieldType: type of the Field which incurred the error
133 - fieldName: name of the Field which incurred the error
134 - fullname: fully qualified name of the Field instance
135 - history: full history of all changes to the Field instance
136 - fieldSource: file and line number of the Field definition
141 self.
fullname = _joinNamePath(config._name, field.name)
142 self.
history = config.history.setdefault(field.name, [])
145 error =
"%s '%s' failed validation: %s\n"\
146 "For more information read the Field definition at:\n%s"\
147 "And the Config definition at:\n%s" % \
148 (self.fieldType.__name__, self.
fullname, msg,
151 ValueError.__init__(self, error)
155 """A field in a a Config.
157 Instances of Field should be class attributes of Config subclasses:
158 Field only supports basic data types (int, float, complex, bool, str)
160 class Example(Config):
161 myInt = Field(int, "an integer field!", default=0)
165 supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex))
167 def __init__(self, doc, dtype, default=None, check=None, optional=False):
168 """Initialize a Field.
170 dtype ------ Data type for the field.
171 doc -------- Documentation for the field.
172 default ---- A default value for the field.
173 check ------ A callable to be called with the field value that returns
174 False if the value is invalid. More complex inter-field
175 validation can be written as part of Config validate()
176 method; this will be ignored if set to None.
177 optional --- When False, Config validate() will fail if value is None
180 raise ValueError(
"Unsupported Field dtype %s" % _typeStr(dtype))
184 dtype = oldStringType
186 source = traceback.extract_stack(limit=2)[0]
187 self.
_setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
189 def _setup(self, doc, dtype, default, check, optional, source):
191 Convenience function provided to simplify initialization of derived
204 Rename an instance of this field, not the field itself.
205 This is invoked by the owning config object and should not be called
208 Only useful for fields which hold sub-configs.
209 Fields which hold subconfigs should rename each sub-config with
210 the full field name as generated by _joinNamePath
216 Base validation for any field.
217 Ensures that non-optional fields are not None.
218 Ensures type correctness
219 Ensures that user-provided check function is valid
220 Most derived Field types should call Field.validate if they choose
221 to re-implement validate
223 value = self.__get__(instance)
224 if not self.optional
and value
is None:
225 raise FieldValidationError(self, instance,
"Required value cannot be None")
229 Make this field read-only.
230 Only important for fields which hold sub-configs.
231 Fields which hold subconfigs should freeze each sub-config.
235 def _validateValue(self, value):
237 Validate a value that is not None
239 This is called from __set__
240 This is not part of the Field API. However, simple derived field types
241 may benifit from implementing _validateValue
246 if not isinstance(value, self.dtype):
247 msg =
"Value %s is of incorrect type %s. Expected type %s" % \
248 (value, _typeStr(value), _typeStr(self.dtype))
250 if self.check
is not None and not self.check(value):
251 msg =
"Value %s is not a valid value" % str(value)
252 raise ValueError(msg)
254 def save(self, outfile, instance):
256 Saves an instance of this field to file.
257 This is invoked by the owning config object, and should not be called
260 outfile ---- an open output stream.
263 fullname = _joinNamePath(instance._name, self.name)
266 doc =
"# " + str(self.
doc).replace(
"\n",
"\n# ")
267 if isinstance(value, float)
and (math.isinf(value)
or math.isnan(value)):
269 outfile.write(
u"{}\n{}=float('{!r}')\n\n".
format(doc, fullname, value))
271 outfile.write(
u"{}\n{}={!r}\n\n".
format(doc, fullname, value))
275 Convert the field value so that it can be set as the value of an item
277 This is invoked by the owning config object and should not be called
280 Simple values are passed through. Complex data structures must be
281 manipulated. For example, a field holding a sub-config should, instead
282 of the subconfig object, return a dict where the keys are the field
283 names in the subconfig, and the values are the field values in the
288 def __get__(self, instance, owner=None, at=None, label="default"):
290 Define how attribute access should occur on the Config instance
291 This is invoked by the owning config object and should not be called
294 When the field attribute is accessed on a Config class object, it
295 returns the field object itself in order to allow inspection of
298 When the field attribute is access on a config instance, the actual
299 value described by the field (and held by the Config instance) is
302 if instance
is None or not isinstance(instance, Config):
305 return instance._storage[self.name]
307 def __set__(self, instance, value, at=None, label='assignment'):
309 Describe how attribute setting should occur on the config instance.
310 This is invoked by the owning config object and should not be called
313 Derived Field classes may need to override the behavior. When overriding
314 __set__, Field authors should follow the following rules:
315 * Do not allow modification of frozen configs
316 * Validate the new value *BEFORE* modifying the field. Except if the
317 new value is None. None is special and no attempt should be made to
318 validate it until Config.validate is called.
319 * Do not modify the Config instance to contain invalid values.
320 * If the field is modified, update the history of the field to reflect the
323 In order to decrease the need to implement this method in derived Field
324 types, value validation is performed in the method _validateValue. If
325 only the validation step differs in the derived Field, it is simpler to
326 implement _validateValue than to re-implement __set__. More complicated
327 behavior, however, may require a reimplementation.
332 history = instance._history.setdefault(self.name, [])
333 if value
is not None:
334 value = _autocast(value, self.
dtype)
337 except BaseException
as e:
340 instance._storage[self.name] = value
342 at = traceback.extract_stack()[:-1]
343 history.append((value, at, label))
347 Describe how attribute deletion should occur on the Config instance.
348 This is invoked by the owning config object and should not be called
352 at = traceback.extract_stack()[:-1]
353 self.
__set__(instance,
None, at=at, label=label)
355 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
356 """Helper function for Config.compare; used to compare two fields for equality.
358 Must be overridden by more complex field types.
360 @param[in] instance1 LHS Config instance to compare.
361 @param[in] instance2 RHS Config instance to compare.
362 @param[in] shortcut If True, return as soon as an inequality is found.
363 @param[in] rtol Relative tolerance for floating point comparisons.
364 @param[in] atol Absolute tolerance for floating point comparisons.
365 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly)
366 to report inequalities.
368 Floating point comparisons are performed by numpy.allclose; refer to that for details.
370 v1 = getattr(instance1, self.name)
371 v2 = getattr(instance2, self.name)
373 _joinNamePath(instance1._name, self.name),
374 _joinNamePath(instance2._name, self.name)
380 """An Importer (for sys.meta_path) that records which modules are being imported.
382 Objects also act as Context Managers, so you can:
383 with RecordingImporter() as importer:
385 print("Imported: " + importer.getModules())
386 This ensures it is properly uninstalled when done.
388 This class makes no effort to do any importing itself.
391 """Create and install the Importer"""
397 sys.meta_path = [self] + sys.meta_path
405 """Uninstall the Importer"""
409 """Called as part of the 'import' chain of events.
411 We return None because we don't do any importing.
413 self._modules.add(fullname)
417 """Return the set of modules that were imported."""
422 """Base class for control objects.
424 A Config object will usually have several Field instances as class
425 attributes; these are used to define most of the base class behavior.
426 Simple derived class should be able to be defined simply by setting those
429 Config also emulates a dict of field name: field value
433 """!Iterate over fields
435 return self._fields.__iter__()
438 """!Return the list of field names
440 return list(self._storage.keys())
443 """!Return the list of field values
445 return list(self._storage.values())
448 """!Return the list of (field name, field value) pairs
450 return list(self._storage.items())
453 """!Iterate over (field name, field value) pairs
455 return iter(self._storage.items())
458 """!Iterate over field values
460 return iter(self.storage.values())
463 """!Iterate over field names
465 return iter(self.storage.keys())
468 """!Return True if the specified field exists in this config
470 @param[in] name field name to test for
472 return self._storage.__contains__(name)
475 """!Allocate a new Config object.
477 In order to ensure that all Config object are always in a proper
478 state when handed to users or to derived Config classes, some
479 attributes are handled at allocation time rather than at initialization
481 This ensures that even if a derived Config class implements __init__,
482 the author does not need to be concerned about when or even if he
483 should call the base Config.__init__
485 name = kw.pop(
"__name",
None)
486 at = kw.pop(
"__at", traceback.extract_stack()[:-1])
488 kw.pop(
"__label",
"default")
490 instance = object.__new__(cls)
491 instance._frozen =
False
492 instance._name = name
493 instance._storage = {}
494 instance._history = {}
495 instance._imports = set()
497 for field
in instance._fields.values():
498 instance._history[field.name] = []
499 field.__set__(instance, field.default, at=at+[field.source], label=
"default")
501 instance.setDefaults()
503 instance.update(__at=at, **kw)
507 """Reduction for pickling (function with arguments to reproduce).
509 We need to condense and reconstitute the Config, since it may contain lambdas
510 (as the 'check' elements) that cannot be pickled.
513 stream = io.StringIO()
515 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
519 Derived config classes that must compute defaults rather than using the
520 Field defaults should do so here.
521 To correctly use inherited defaults, implementations of setDefaults()
522 must call their base class' setDefaults()
527 """!Update values specified by the keyword arguments
529 @warning The '__at' and '__label' keyword arguments are special internal
530 keywords. They are used to strip out any internal steps from the
531 history tracebacks of the config. Modifying these keywords allows users
532 to lie about a Config's history. Please do not do so!
534 at = kw.pop(
"__at", traceback.extract_stack()[:-1])
535 label = kw.pop(
"__label",
"update")
537 for name, value
in kw.items():
539 field = self._fields[name]
540 field.__set__(self, value, at=at, label=label)
542 raise KeyError(
"No field of name %s exists in config type %s" % (name, _typeStr(self)))
544 def load(self, filename, root="config"):
545 """!Modify this config in place by executing the Python code in the named file.
547 @param[in] filename name of file containing config override code
548 @param[in] root name of variable in file that refers to the config being overridden
550 For example: if the value of root is "config" and the file contains this text:
551 "config.myField = 5" then this config's field "myField" is set to 5.
553 @deprecated For purposes of backwards compatibility, older config files that use
554 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr.
555 This feature will be removed at some point.
557 with open(filename,
"r") as f:
558 code = compile(f.read(), filename=filename, mode="exec")
562 """!Modify this config in place by executing the python code in the provided stream.
564 @param[in] stream open file object, string or compiled string containing config override code
565 @param[in] root name of variable in stream that refers to the config being overridden
566 @param[in] filename name of config override file, or None if unknown or contained
567 in the stream; used for error reporting
569 For example: if the value of root is "config" and the stream contains this text:
570 "config.myField = 5" then this config's field "myField" is set to 5.
572 @deprecated For purposes of backwards compatibility, older config files that use
573 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr.
574 This feature will be removed at some point.
579 exec(stream, {}, local)
580 except NameError
as e:
581 if root ==
"config" and "root" in e.args[0]:
585 filename = getattr(stream,
"co_filename",
None)
587 filename = getattr(stream,
"name",
"?")
588 sys.stderr.write(
u"Config override file %r" % (filename,) +
589 u" appears to use 'root' instead of 'config'; trying with 'root'")
590 local = {
"root": self}
591 exec(stream, {}, local)
595 self._imports.update(importer.getModules())
597 def save(self, filename, root="config"):
598 """!Save a python script to the named file, which, when loaded, reproduces this Config
600 @param[in] filename name of file to which to write the config
601 @param[in] root name to use for the root config variable; the same value must be used when loading
603 d = os.path.dirname(filename)
604 with tempfile.NamedTemporaryFile(mode=
"w", delete=
False, dir=d)
as outfile:
609 umask = os.umask(0o077)
611 os.chmod(outfile.name, (~umask & 0o666))
615 shutil.move(outfile.name, filename)
618 """!Save a python script to a stream, which, when loaded, reproduces this Config
620 @param outfile [inout] open file object to which to write the config. Accepts strings not bytes.
621 @param root [in] name to use for the root config variable; the same value must be used when loading
626 configType = type(self)
627 typeString = _typeStr(configType)
628 outfile.write(
u"import {}\n".
format(configType.__module__))
629 outfile.write(
u"assert type({})=={}, 'config is of type %s.%s ".
format(root, typeString))
630 outfile.write(
u"instead of {}' % (type({}).__module__, type({}).__name__)\n".
format(typeString,
638 """!Make this Config and all sub-configs read-only
641 for field
in self._fields.values():
644 def _save(self, outfile):
645 """!Save this Config to an open stream object
647 for imp
in self._imports:
648 if imp
in sys.modules
and sys.modules[imp]
is not None:
649 outfile.write(
u"import {}\n".
format(imp))
650 for field
in self._fields.values():
651 field.save(outfile, self)
654 """!Return a dict of field name: value
656 Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new
657 Field type, you may need to implement your own toDict method.
660 for name, field
in self._fields.items():
661 dict_[name] = field.toDict(self)
664 def _rename(self, name):
665 """!Rename this Config object in its parent config
667 @param[in] name new name for this config in its parent config
669 Correct behavior is dependent on proper implementation of Field.rename. If implementing a new
670 Field type, you may need to implement your own rename method.
673 for field
in self._fields.values():
677 """!Validate the Config; raise an exception if invalid
679 The base class implementation performs type checks on all fields by
680 calling Field.validate().
682 Complex single-field validation can be defined by deriving new Field
683 types. As syntactic sugar, some derived Field types are defined in
684 this module which handle recursing into sub-configs
685 (ConfigField, ConfigChoiceField)
687 Inter-field relationships should only be checked in derived Config
688 classes after calling this method, and base validation is complete
690 for field
in self._fields.values():
694 """!Format the specified config field's history to a more human-readable format
696 @param[in] name name of field whose history is wanted
697 @param[in] kwargs keyword arguments for lsst.pex.config.history.format
698 @return a string containing the formatted history
701 return pexHist.format(self, name, **kwargs)
704 Read-only history property
706 history = property(
lambda x: x._history)
709 """!Regulate which attributes can be set
711 Unlike normal python objects, Config objects are locked such
712 that no additional attributes nor properties may be added to them
715 Although this is not the standard Python behavior, it helps to
716 protect users from accidentally mispelling a field name, or
717 trying to set a non-existent field.
721 at = traceback.extract_stack()[:-1]
723 self.
_fields[attr].__set__(self, value, at=at, label=label)
724 elif hasattr(getattr(self.__class__, attr,
None),
'__set__'):
726 return object.__setattr__(self, attr, value)
727 elif attr
in self.__dict__
or attr
in (
"_name",
"_history",
"_storage",
"_frozen",
"_imports"):
729 self.__dict__[attr] = value
732 raise AttributeError(
"%s has no attribute %s" % (_typeStr(self), attr))
737 at = traceback.extract_stack()[:-1]
738 self.
_fields[attr].__delete__(self, at=at, label=label)
740 object.__delattr__(self, attr)
743 if type(other) == type(self):
745 thisValue = getattr(self, name)
746 otherValue = getattr(other, name)
747 if isinstance(thisValue, float)
and math.isnan(thisValue):
748 if not math.isnan(otherValue):
750 elif thisValue != otherValue:
756 return not self.
__eq__(other)
764 ", ".join(
"%s=%r" % (k, v)
for k, v
in self.
toDict().
items()
if v
is not None)
767 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
768 """!Compare two Configs for equality; return True if equal
770 If the Configs contain RegistryFields or ConfigChoiceFields, unselected Configs
771 will not be compared.
773 @param[in] other Config object to compare with self.
774 @param[in] shortcut If True, return as soon as an inequality is found.
775 @param[in] rtol Relative tolerance for floating point comparisons.
776 @param[in] atol Absolute tolerance for floating point comparisons.
777 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly)
778 to report inequalities.
780 Floating point comparisons are performed by numpy.allclose; refer to that for details.
782 name1 = self.
_name if self.
_name is not None else "config"
783 name2 = other._name
if other._name
is not None else "config"
786 rtol=rtol, atol=atol, output=output)
791 config.loadFromStream(stream)
def keys
Return the list of field names.
def __setattr__
Regulate which attributes can be set.
def freeze
Make this Config and all sub-configs read-only.
def saveToStream
Save a python script to a stream, which, when loaded, reproduces this Config.
def loadFromStream
Modify this config in place by executing the python code in the provided stream.
def compare
Compare two Configs for equality; return True if equal.
def iteritems
Iterate over (field name, field value) pairs.
def __contains__
Return True if the specified field exists in this config.
def save
Save a python script to the named file, which, when loaded, reproduces this Config.
def _rename
Rename this Config object in its parent config.
def _save
Save this Config to an open stream object.
def load
Modify this config in place by executing the Python code in the named file.
def itervalues
Iterate over field values.
def iterkeys
Iterate over field names.
def toDict
Return a dict of field name: value.
def __iter__
Iterate over fields.
def formatHistory
Format the specified config field's history to a more human-readable format.
def items
Return the list of (field name, field value) pairs.
def update
Update values specified by the keyword arguments.
def validate
Validate the Config; raise an exception if invalid.
def __new__
Allocate a new Config object.
def values
Return the list of field values.