23 from builtins
import str
24 from builtins
import object
25 from past.builtins
import long
26 from past.builtins
import basestring
27 from past.builtins
import unicode
37 from .comparison
import getComparisonName, compareScalars, compareConfigs
38 from .callStack
import getStackFrame, getCallStack
39 from future.utils
import with_metaclass
41 __all__ = (
"Config",
"Field",
"FieldValidationError")
44 def _joinNamePath(prefix=None, name=None, index=None):
46 Utility function for generating nested configuration names
48 if not prefix
and not name:
49 raise ValueError(
"Invalid name: cannot be None")
53 name = prefix +
"." + name
56 return "%s[%r]" % (name, index)
61 def _autocast(x, dtype):
63 If appropriate perform type casting of value x to type dtype,
64 otherwise return the original value x
66 if dtype == float
and isinstance(x, int):
68 if dtype == int
and isinstance(x, long):
70 if isinstance(x, str):
77 Utility function to generate a fully qualified type name.
79 This is used primarily in writing config files to be
80 executed later upon 'load'.
82 if hasattr(x,
'__module__')
and hasattr(x,
'__name__'):
86 if (sys.version_info.major <= 2
and xtype.__module__ ==
'__builtin__')
or xtype.__module__ ==
'builtins':
89 return "%s.%s" % (xtype.__module__, xtype.__name__)
93 """A metaclass for Config
95 Adds a dictionary containing all Field class attributes
96 as a class attribute called '_fields', and adds the name of each field as
97 an instance variable of the field itself (so you don't have to pass the
98 name of the field to the field constructor).
101 type.__init__(self, name, bases, dict_)
105 def getFields(classtype):
107 bases = list(classtype.__bases__)
110 fields.update(getFields(b))
112 for k, v
in classtype.__dict__.items():
113 if isinstance(v, Field):
117 fields = getFields(self)
118 for k, v
in fields.items():
119 setattr(self, k, copy.deepcopy(v))
122 if isinstance(value, Field):
125 type.__setattr__(self, name, value)
130 Custom exception class which holds additional information useful to
131 debuggin Config errors:
132 - fieldType: type of the Field which incurred the error
133 - fieldName: name of the Field which incurred the error
134 - fullname: fully qualified name of the Field instance
135 - history: full history of all changes to the Field instance
136 - fieldSource: file and line number of the Field definition
141 self.
fullname = _joinNamePath(config._name, field.name)
142 self.
history = config.history.setdefault(field.name, [])
145 error =
"%s '%s' failed validation: %s\n"\
146 "For more information read the Field definition at:\n%s"\
147 "And the Config definition at:\n%s" % \
148 (self.fieldType.__name__, self.
fullname, msg,
149 self.fieldSource.format(), self.configSource.format())
150 ValueError.__init__(self, error)
154 """A field in a a Config.
156 Instances of Field should be class attributes of Config subclasses:
157 Field only supports basic data types (int, float, complex, bool, str)
159 class Example(Config):
160 myInt = Field(int, "an integer field!", default=0)
164 supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex))
166 def __init__(self, doc, dtype, default=None, check=None, optional=False):
167 """Initialize a Field.
169 dtype ------ Data type for the field.
170 doc -------- Documentation for the field.
171 default ---- A default value for the field.
172 check ------ A callable to be called with the field value that returns
173 False if the value is invalid. More complex inter-field
174 validation can be written as part of Config validate()
175 method; this will be ignored if set to None.
176 optional --- When False, Config validate() will fail if value is None
179 raise ValueError(
"Unsupported Field dtype %s" % _typeStr(dtype))
183 dtype = oldStringType
186 self.
_setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
188 def _setup(self, doc, dtype, default, check, optional, source):
190 Convenience function provided to simplify initialization of derived
203 Rename an instance of this field, not the field itself.
204 This is invoked by the owning config object and should not be called
207 Only useful for fields which hold sub-configs.
208 Fields which hold subconfigs should rename each sub-config with
209 the full field name as generated by _joinNamePath
215 Base validation for any field.
216 Ensures that non-optional fields are not None.
217 Ensures type correctness
218 Ensures that user-provided check function is valid
219 Most derived Field types should call Field.validate if they choose
220 to re-implement validate
222 value = self.__get__(instance)
223 if not self.optional
and value
is None:
224 raise FieldValidationError(self, instance,
"Required value cannot be None")
228 Make this field read-only.
229 Only important for fields which hold sub-configs.
230 Fields which hold subconfigs should freeze each sub-config.
234 def _validateValue(self, value):
236 Validate a value that is not None
238 This is called from __set__
239 This is not part of the Field API. However, simple derived field types
240 may benifit from implementing _validateValue
245 if not isinstance(value, self.dtype):
246 msg =
"Value %s is of incorrect type %s. Expected type %s" % \
247 (value, _typeStr(value), _typeStr(self.dtype))
249 if self.check
is not None and not self.check(value):
250 msg =
"Value %s is not a valid value" % str(value)
251 raise ValueError(msg)
253 def save(self, outfile, instance):
255 Saves an instance of this field to file.
256 This is invoked by the owning config object, and should not be called
259 outfile ---- an open output stream.
262 fullname = _joinNamePath(instance._name, self.name)
265 doc =
"# " + str(self.
doc).replace(
"\n",
"\n# ")
266 if isinstance(value, float)
and (math.isinf(value)
or math.isnan(value)):
268 outfile.write(
u"{}\n{}=float('{!r}')\n\n".
format(doc, fullname, value))
270 outfile.write(
u"{}\n{}={!r}\n\n".
format(doc, fullname, value))
274 Convert the field value so that it can be set as the value of an item
276 This is invoked by the owning config object and should not be called
279 Simple values are passed through. Complex data structures must be
280 manipulated. For example, a field holding a sub-config should, instead
281 of the subconfig object, return a dict where the keys are the field
282 names in the subconfig, and the values are the field values in the
287 def __get__(self, instance, owner=None, at=None, label="default"):
289 Define how attribute access should occur on the Config instance
290 This is invoked by the owning config object and should not be called
293 When the field attribute is accessed on a Config class object, it
294 returns the field object itself in order to allow inspection of
297 When the field attribute is access on a config instance, the actual
298 value described by the field (and held by the Config instance) is
301 if instance
is None or not isinstance(instance, Config):
304 return instance._storage[self.name]
306 def __set__(self, instance, value, at=None, label='assignment'):
308 Describe how attribute setting should occur on the config instance.
309 This is invoked by the owning config object and should not be called
312 Derived Field classes may need to override the behavior. When overriding
313 __set__, Field authors should follow the following rules:
314 * Do not allow modification of frozen configs
315 * Validate the new value *BEFORE* modifying the field. Except if the
316 new value is None. None is special and no attempt should be made to
317 validate it until Config.validate is called.
318 * Do not modify the Config instance to contain invalid values.
319 * If the field is modified, update the history of the field to reflect the
322 In order to decrease the need to implement this method in derived Field
323 types, value validation is performed in the method _validateValue. If
324 only the validation step differs in the derived Field, it is simpler to
325 implement _validateValue than to re-implement __set__. More complicated
326 behavior, however, may require a reimplementation.
331 history = instance._history.setdefault(self.name, [])
332 if value
is not None:
333 value = _autocast(value, self.
dtype)
336 except BaseException
as e:
339 instance._storage[self.name] = value
342 history.append((value, at, label))
346 Describe how attribute deletion should occur on the Config instance.
347 This is invoked by the owning config object and should not be called
352 self.
__set__(instance,
None, at=at, label=label)
354 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
355 """Helper function for Config.compare; used to compare two fields for equality.
357 Must be overridden by more complex field types.
359 @param[in] instance1 LHS Config instance to compare.
360 @param[in] instance2 RHS Config instance to compare.
361 @param[in] shortcut If True, return as soon as an inequality is found.
362 @param[in] rtol Relative tolerance for floating point comparisons.
363 @param[in] atol Absolute tolerance for floating point comparisons.
364 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly)
365 to report inequalities.
367 Floating point comparisons are performed by numpy.allclose; refer to that for details.
369 v1 = getattr(instance1, self.name)
370 v2 = getattr(instance2, self.name)
372 _joinNamePath(instance1._name, self.name),
373 _joinNamePath(instance2._name, self.name)
379 """An Importer (for sys.meta_path) that records which modules are being imported.
381 Objects also act as Context Managers, so you can:
382 with RecordingImporter() as importer:
384 print("Imported: " + importer.getModules())
385 This ensures it is properly uninstalled when done.
387 This class makes no effort to do any importing itself.
390 """Create and install the Importer"""
396 sys.meta_path = [self] + sys.meta_path
404 """Uninstall the Importer"""
408 """Called as part of the 'import' chain of events.
410 We return None because we don't do any importing.
412 self._modules.add(fullname)
416 """Return the set of modules that were imported."""
421 """Base class for control objects.
423 A Config object will usually have several Field instances as class
424 attributes; these are used to define most of the base class behavior.
425 Simple derived class should be able to be defined simply by setting those
428 Config also emulates a dict of field name: field value
432 """!Iterate over fields
434 return self._fields.__iter__()
437 """!Return the list of field names
439 return list(self._storage.keys())
442 """!Return the list of field values
444 return list(self._storage.values())
447 """!Return the list of (field name, field value) pairs
449 return list(self._storage.items())
452 """!Iterate over (field name, field value) pairs
454 return iter(self._storage.items())
457 """!Iterate over field values
459 return iter(self.storage.values())
462 """!Iterate over field names
464 return iter(self.storage.keys())
467 """!Return True if the specified field exists in this config
469 @param[in] name field name to test for
471 return self._storage.__contains__(name)
474 """!Allocate a new Config object.
476 In order to ensure that all Config object are always in a proper
477 state when handed to users or to derived Config classes, some
478 attributes are handled at allocation time rather than at initialization
480 This ensures that even if a derived Config class implements __init__,
481 the author does not need to be concerned about when or even if he
482 should call the base Config.__init__
484 name = kw.pop(
"__name",
None)
487 kw.pop(
"__label",
"default")
489 instance = object.__new__(cls)
490 instance._frozen =
False
491 instance._name = name
492 instance._storage = {}
493 instance._history = {}
494 instance._imports = set()
496 for field
in instance._fields.values():
497 instance._history[field.name] = []
498 field.__set__(instance, field.default, at=at + [field.source], label=
"default")
500 instance.setDefaults()
502 instance.update(__at=at, **kw)
506 """Reduction for pickling (function with arguments to reproduce).
508 We need to condense and reconstitute the Config, since it may contain lambdas
509 (as the 'check' elements) that cannot be pickled.
512 stream = io.StringIO()
514 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
518 Derived config classes that must compute defaults rather than using the
519 Field defaults should do so here.
520 To correctly use inherited defaults, implementations of setDefaults()
521 must call their base class' setDefaults()
526 """!Update values specified by the keyword arguments
528 @warning The '__at' and '__label' keyword arguments are special internal
529 keywords. They are used to strip out any internal steps from the
530 history tracebacks of the config. Modifying these keywords allows users
531 to lie about a Config's history. Please do not do so!
534 label = kw.pop(
"__label",
"update")
536 for name, value
in kw.items():
538 field = self._fields[name]
539 field.__set__(self, value, at=at, label=label)
541 raise KeyError(
"No field of name %s exists in config type %s" % (name, _typeStr(self)))
543 def load(self, filename, root="config"):
544 """!Modify this config in place by executing the Python code in the named file.
546 @param[in] filename name of file containing config override code
547 @param[in] root name of variable in file that refers to the config being overridden
549 For example: if the value of root is "config" and the file contains this text:
550 "config.myField = 5" then this config's field "myField" is set to 5.
552 @deprecated For purposes of backwards compatibility, older config files that use
553 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr.
554 This feature will be removed at some point.
556 with open(filename,
"r") as f:
557 code = compile(f.read(), filename=filename, mode="exec")
561 """!Modify this config in place by executing the python code in the provided stream.
563 @param[in] stream open file object, string or compiled string containing config override code
564 @param[in] root name of variable in stream that refers to the config being overridden
565 @param[in] filename name of config override file, or None if unknown or contained
566 in the stream; used for error reporting
568 For example: if the value of root is "config" and the stream contains this text:
569 "config.myField = 5" then this config's field "myField" is set to 5.
571 @deprecated For purposes of backwards compatibility, older config files that use
572 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr.
573 This feature will be removed at some point.
578 exec(stream, {}, local)
579 except NameError
as e:
580 if root ==
"config" and "root" in e.args[0]:
584 filename = getattr(stream,
"co_filename",
None)
586 filename = getattr(stream,
"name",
"?")
587 sys.stderr.write(
u"Config override file %r" % (filename,) +
588 u" appears to use 'root' instead of 'config'; trying with 'root'")
589 local = {
"root": self}
590 exec(stream, {}, local)
594 self._imports.update(importer.getModules())
596 def save(self, filename, root="config"):
597 """!Save a python script to the named file, which, when loaded, reproduces this Config
599 @param[in] filename name of file to which to write the config
600 @param[in] root name to use for the root config variable; the same value must be used when loading
602 d = os.path.dirname(filename)
603 with tempfile.NamedTemporaryFile(mode=
"w", delete=
False, dir=d)
as outfile:
608 umask = os.umask(0o077)
610 os.chmod(outfile.name, (~umask & 0o666))
614 shutil.move(outfile.name, filename)
617 """!Save a python script to a stream, which, when loaded, reproduces this Config
619 @param outfile [inout] open file object to which to write the config. Accepts strings not bytes.
620 @param root [in] name to use for the root config variable; the same value must be used when loading
625 configType = type(self)
626 typeString = _typeStr(configType)
627 outfile.write(
u"import {}\n".
format(configType.__module__))
628 outfile.write(
u"assert type({})=={}, 'config is of type %s.%s ".
format(root, typeString))
629 outfile.write(
u"instead of {}' % (type({}).__module__, type({}).__name__)\n".
format(typeString,
637 """!Make this Config and all sub-configs read-only
640 for field
in self._fields.values():
643 def _save(self, outfile):
644 """!Save this Config to an open stream object
646 for imp
in self._imports:
647 if imp
in sys.modules
and sys.modules[imp]
is not None:
648 outfile.write(
u"import {}\n".
format(imp))
649 for field
in self._fields.values():
650 field.save(outfile, self)
653 """!Return a dict of field name: value
655 Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new
656 Field type, you may need to implement your own toDict method.
659 for name, field
in self._fields.items():
660 dict_[name] = field.toDict(self)
663 def _rename(self, name):
664 """!Rename this Config object in its parent config
666 @param[in] name new name for this config in its parent config
668 Correct behavior is dependent on proper implementation of Field.rename. If implementing a new
669 Field type, you may need to implement your own rename method.
672 for field
in self._fields.values():
676 """!Validate the Config; raise an exception if invalid
678 The base class implementation performs type checks on all fields by
679 calling Field.validate().
681 Complex single-field validation can be defined by deriving new Field
682 types. As syntactic sugar, some derived Field types are defined in
683 this module which handle recursing into sub-configs
684 (ConfigField, ConfigChoiceField)
686 Inter-field relationships should only be checked in derived Config
687 classes after calling this method, and base validation is complete
689 for field
in self._fields.values():
693 """!Format the specified config field's history to a more human-readable format
695 @param[in] name name of field whose history is wanted
696 @param[in] kwargs keyword arguments for lsst.pex.config.history.format
697 @return a string containing the formatted history
700 return pexHist.format(self, name, **kwargs)
703 Read-only history property
705 history = property(
lambda x: x._history)
708 """!Regulate which attributes can be set
710 Unlike normal python objects, Config objects are locked such
711 that no additional attributes nor properties may be added to them
714 Although this is not the standard Python behavior, it helps to
715 protect users from accidentally mispelling a field name, or
716 trying to set a non-existent field.
722 self.
_fields[attr].__set__(self, value, at=at, label=label)
723 elif hasattr(getattr(self.__class__, attr,
None),
'__set__'):
725 return object.__setattr__(self, attr, value)
726 elif attr
in self.__dict__
or attr
in (
"_name",
"_history",
"_storage",
"_frozen",
"_imports"):
728 self.__dict__[attr] = value
731 raise AttributeError(
"%s has no attribute %s" % (_typeStr(self), attr))
737 self.
_fields[attr].__delete__(self, at=at, label=label)
739 object.__delattr__(self, attr)
742 if type(other) == type(self):
744 thisValue = getattr(self, name)
745 otherValue = getattr(other, name)
746 if isinstance(thisValue, float)
and math.isnan(thisValue):
747 if not math.isnan(otherValue):
749 elif thisValue != otherValue:
755 return not self.
__eq__(other)
763 ", ".join(
"%s=%r" % (k, v)
for k, v
in self.
toDict().
items()
if v
is not None)
766 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
767 """!Compare two Configs for equality; return True if equal
769 If the Configs contain RegistryFields or ConfigChoiceFields, unselected Configs
770 will not be compared.
772 @param[in] other Config object to compare with self.
773 @param[in] shortcut If True, return as soon as an inequality is found.
774 @param[in] rtol Relative tolerance for floating point comparisons.
775 @param[in] atol Absolute tolerance for floating point comparisons.
776 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly)
777 to report inequalities.
779 Floating point comparisons are performed by numpy.allclose; refer to that for details.
781 name1 = self.
_name if self.
_name is not None else "config"
782 name2 = other._name
if other._name
is not None else "config"
785 rtol=rtol, atol=atol, output=output)
790 config.loadFromStream(stream)
def keys
Return the list of field names.
def __setattr__
Regulate which attributes can be set.
def freeze
Make this Config and all sub-configs read-only.
def saveToStream
Save a python script to a stream, which, when loaded, reproduces this Config.
def loadFromStream
Modify this config in place by executing the python code in the provided stream.
def compare
Compare two Configs for equality; return True if equal.
def iteritems
Iterate over (field name, field value) pairs.
def __contains__
Return True if the specified field exists in this config.
def save
Save a python script to the named file, which, when loaded, reproduces this Config.
def _rename
Rename this Config object in its parent config.
def _save
Save this Config to an open stream object.
def load
Modify this config in place by executing the Python code in the named file.
def itervalues
Iterate over field values.
def iterkeys
Iterate over field names.
def toDict
Return a dict of field name: value.
def __iter__
Iterate over fields.
def formatHistory
Format the specified config field's history to a more human-readable format.
def items
Return the list of (field name, field value) pairs.
def update
Update values specified by the keyword arguments.
def validate
Validate the Config; raise an exception if invalid.
def __new__
Allocate a new Config object.
def values
Return the list of field values.