23 from builtins
import str
24 from builtins
import object
25 from past.builtins
import long
26 from past.builtins
import basestring
27 from past.builtins
import unicode
38 from .comparison
import getComparisonName, compareScalars, compareConfigs
39 from .callStack
import getStackFrame, getCallStack
40 from future.utils
import with_metaclass
42 __all__ = (
"Config",
"Field",
"FieldValidationError")
45 def _joinNamePath(prefix=None, name=None, index=None):
47 Utility function for generating nested configuration names 49 if not prefix
and not name:
50 raise ValueError(
"Invalid name: cannot be None")
54 name = prefix +
"." + name
57 return "%s[%r]" % (name, index)
62 def _autocast(x, dtype):
64 If appropriate perform type casting of value x to type dtype, 65 otherwise return the original value x 67 if dtype == float
and isinstance(x, int):
69 if dtype == int
and isinstance(x, long):
71 if isinstance(x, str):
78 Utility function to generate a fully qualified type name. 80 This is used primarily in writing config files to be 81 executed later upon 'load'. 83 if hasattr(x,
'__module__')
and hasattr(x,
'__name__'):
87 if (sys.version_info.major <= 2
and xtype.__module__ ==
'__builtin__')
or xtype.__module__ ==
'builtins':
90 return "%s.%s" % (xtype.__module__, xtype.__name__)
94 """A metaclass for Config 96 Adds a dictionary containing all Field class attributes 97 as a class attribute called '_fields', and adds the name of each field as 98 an instance variable of the field itself (so you don't have to pass the 99 name of the field to the field constructor). 102 type.__init__(self, name, bases, dict_)
106 def getFields(classtype):
108 bases = list(classtype.__bases__)
111 fields.update(getFields(b))
113 for k, v
in classtype.__dict__.items():
114 if isinstance(v, Field):
118 fields = getFields(self)
119 for k, v
in fields.items():
120 setattr(self, k, copy.deepcopy(v))
123 if isinstance(value, Field):
126 type.__setattr__(self, name, value)
131 Custom exception class which holds additional information useful to 132 debuggin Config errors: 133 - fieldType: type of the Field which incurred the error 134 - fieldName: name of the Field which incurred the error 135 - fullname: fully qualified name of the Field instance 136 - history: full history of all changes to the Field instance 137 - fieldSource: file and line number of the Field definition 142 self.
fullname = _joinNamePath(config._name, field.name)
143 self.
history = config.history.setdefault(field.name, [])
146 error =
"%s '%s' failed validation: %s\n"\
147 "For more information read the Field definition at:\n%s"\
148 "And the Config definition at:\n%s" % \
151 ValueError.__init__(self, error)
155 """A field in a a Config. 157 Instances of Field should be class attributes of Config subclasses: 158 Field only supports basic data types (int, float, complex, bool, str) 160 class Example(Config): 161 myInt = Field(int, "an integer field!", default=0) 165 supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex))
167 def __init__(self, doc, dtype, default=None, check=None, optional=False):
168 """Initialize a Field. 170 dtype ------ Data type for the field. 171 doc -------- Documentation for the field. 172 default ---- A default value for the field. 173 check ------ A callable to be called with the field value that returns 174 False if the value is invalid. More complex inter-field 175 validation can be written as part of Config validate() 176 method; this will be ignored if set to None. 177 optional --- When False, Config validate() will fail if value is None 180 raise ValueError(
"Unsupported Field dtype %s" % _typeStr(dtype))
184 dtype = oldStringType
187 self.
_setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
189 def _setup(self, doc, dtype, default, check, optional, source):
191 Convenience function provided to simplify initialization of derived 196 self.
__doc__ = doc+
" (`"+dtype.__name__+
"`, default "+
'``{0!r}``'.
format(default)+
")" 204 Rename an instance of this field, not the field itself. 205 This is invoked by the owning config object and should not be called 208 Only useful for fields which hold sub-configs. 209 Fields which hold subconfigs should rename each sub-config with 210 the full field name as generated by _joinNamePath 216 Base validation for any field. 217 Ensures that non-optional fields are not None. 218 Ensures type correctness 219 Ensures that user-provided check function is valid 220 Most derived Field types should call Field.validate if they choose 221 to re-implement validate 223 value = self.__get__(instance)
224 if not self.optional
and value
is None:
225 raise FieldValidationError(self, instance,
"Required value cannot be None")
229 Make this field read-only. 230 Only important for fields which hold sub-configs. 231 Fields which hold subconfigs should freeze each sub-config. 235 def _validateValue(self, value):
237 Validate a value that is not None 239 This is called from __set__ 240 This is not part of the Field API. However, simple derived field types 241 may benefit from implementing _validateValue 246 if not isinstance(value, self.dtype):
247 msg =
"Value %s is of incorrect type %s. Expected type %s" % \
248 (value, _typeStr(value), _typeStr(self.dtype))
250 if self.check
is not None and not self.check(value):
251 msg =
"Value %s is not a valid value" % str(value)
252 raise ValueError(msg)
254 def save(self, outfile, instance):
256 Saves an instance of this field to file. 257 This is invoked by the owning config object, and should not be called 260 outfile ---- an open output stream. 263 fullname = _joinNamePath(instance._name, self.name)
266 doc =
"# " + str(self.
doc).replace(
"\n",
"\n# ")
267 if isinstance(value, float)
and (math.isinf(value)
or math.isnan(value)):
269 outfile.write(
u"{}\n{}=float('{!r}')\n\n".
format(doc, fullname, value))
271 outfile.write(
u"{}\n{}={!r}\n\n".
format(doc, fullname, value))
275 Convert the field value so that it can be set as the value of an item 277 This is invoked by the owning config object and should not be called 280 Simple values are passed through. Complex data structures must be 281 manipulated. For example, a field holding a sub-config should, instead 282 of the subconfig object, return a dict where the keys are the field 283 names in the subconfig, and the values are the field values in the 288 def __get__(self, instance, owner=None, at=None, label="default"):
290 Define how attribute access should occur on the Config instance 291 This is invoked by the owning config object and should not be called 294 When the field attribute is accessed on a Config class object, it 295 returns the field object itself in order to allow inspection of 298 When the field attribute is access on a config instance, the actual 299 value described by the field (and held by the Config instance) is 302 if instance
is None or not isinstance(instance, Config):
305 return instance._storage[self.name]
307 def __set__(self, instance, value, at=None, label='assignment'):
309 Describe how attribute setting should occur on the config instance. 310 This is invoked by the owning config object and should not be called 313 Derived Field classes may need to override the behavior. When overriding 314 __set__, Field authors should follow the following rules: 315 * Do not allow modification of frozen configs 316 * Validate the new value *BEFORE* modifying the field. Except if the 317 new value is None. None is special and no attempt should be made to 318 validate it until Config.validate is called. 319 * Do not modify the Config instance to contain invalid values. 320 * If the field is modified, update the history of the field to reflect the 323 In order to decrease the need to implement this method in derived Field 324 types, value validation is performed in the method _validateValue. If 325 only the validation step differs in the derived Field, it is simpler to 326 implement _validateValue than to re-implement __set__. More complicated 327 behavior, however, may require a reimplementation. 332 history = instance._history.setdefault(self.name, [])
333 if value
is not None:
334 value = _autocast(value, self.
dtype)
337 except BaseException
as e:
340 instance._storage[self.name] = value
343 history.append((value, at, label))
347 Describe how attribute deletion should occur on the Config instance. 348 This is invoked by the owning config object and should not be called 353 self.
__set__(instance,
None, at=at, label=label)
355 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
356 """Helper function for Config.compare; used to compare two fields for equality. 358 Must be overridden by more complex field types. 360 @param[in] instance1 LHS Config instance to compare. 361 @param[in] instance2 RHS Config instance to compare. 362 @param[in] shortcut If True, return as soon as an inequality is found. 363 @param[in] rtol Relative tolerance for floating point comparisons. 364 @param[in] atol Absolute tolerance for floating point comparisons. 365 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 366 to report inequalities. 368 Floating point comparisons are performed by numpy.allclose; refer to that for details. 370 v1 = getattr(instance1, self.name)
371 v2 = getattr(instance2, self.name)
373 _joinNamePath(instance1._name, self.name),
374 _joinNamePath(instance2._name, self.name)
380 """An Importer (for sys.meta_path) that records which modules are being imported. 382 Objects also act as Context Managers, so you can: 383 with RecordingImporter() as importer: 385 print("Imported: " + importer.getModules()) 386 This ensures it is properly uninstalled when done. 388 This class makes no effort to do any importing itself. 391 """Create and install the Importer""" 397 sys.meta_path = [self] + sys.meta_path
405 """Uninstall the Importer""" 409 """Called as part of the 'import' chain of events. 411 We return None because we don't do any importing. 417 """Return the set of modules that were imported.""" 422 """Base class for control objects. 424 A Config object will usually have several Field instances as class 425 attributes; these are used to define most of the base class behavior. 426 Simple derived class should be able to be defined simply by setting those 429 Config also emulates a dict of field name: field value 433 """!Iterate over fields 438 """!Return the list of field names 440 return list(self._storage.
keys())
443 """!Return the list of field values 445 return list(self._storage.
values())
448 """!Return the list of (field name, field value) pairs 450 return list(self._storage.
items())
453 """!Iterate over (field name, field value) pairs 455 return iter(self._storage.
items())
458 """!Iterate over field values 460 return iter(self.storage.
values())
463 """!Iterate over field names 465 return iter(self.storage.
keys())
468 """!Return True if the specified field exists in this config 470 @param[in] name field name to test for 475 """!Allocate a new Config object. 477 In order to ensure that all Config object are always in a proper 478 state when handed to users or to derived Config classes, some 479 attributes are handled at allocation time rather than at initialization 481 This ensures that even if a derived Config class implements __init__, 482 the author does not need to be concerned about when or even if he 483 should call the base Config.__init__ 485 name = kw.pop(
"__name",
None)
488 kw.pop(
"__label",
"default")
490 instance = object.__new__(cls)
491 instance._frozen =
False 492 instance._name = name
493 instance._storage = {}
494 instance._history = {}
495 instance._imports = set()
497 for field
in instance._fields.values():
498 instance._history[field.name] = []
499 field.__set__(instance, field.default, at=at + [field.source], label=
"default")
501 instance.setDefaults()
503 instance.update(__at=at, **kw)
507 """Reduction for pickling (function with arguments to reproduce). 509 We need to condense and reconstitute the Config, since it may contain lambdas 510 (as the 'check' elements) that cannot be pickled. 513 stream = io.StringIO()
515 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
519 Derived config classes that must compute defaults rather than using the 520 Field defaults should do so here. 521 To correctly use inherited defaults, implementations of setDefaults() 522 must call their base class' setDefaults() 527 """!Update values specified by the keyword arguments 529 @warning The '__at' and '__label' keyword arguments are special internal 530 keywords. They are used to strip out any internal steps from the 531 history tracebacks of the config. Modifying these keywords allows users 532 to lie about a Config's history. Please do not do so! 535 label = kw.pop(
"__label",
"update")
537 for name, value
in kw.items():
539 field = self._fields[name]
540 field.__set__(self, value, at=at, label=label)
542 raise KeyError(
"No field of name %s exists in config type %s" % (name, _typeStr(self)))
544 def load(self, filename, root="config"):
545 """!Modify this config in place by executing the Python code in the named file. 547 @param[in] filename name of file containing config override code 548 @param[in] root name of variable in file that refers to the config being overridden 550 For example: if the value of root is "config" and the file contains this text: 551 "config.myField = 5" then this config's field "myField" is set to 5. 553 @deprecated For purposes of backwards compatibility, older config files that use 554 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 555 This feature will be removed at some point. 557 with open(filename,
"r") as f: 558 code = compile(f.read(), filename=filename, mode="exec")
562 """!Modify this config in place by executing the python code in the provided stream. 564 @param[in] stream open file object, string or compiled string containing config override code 565 @param[in] root name of variable in stream that refers to the config being overridden 566 @param[in] filename name of config override file, or None if unknown or contained 567 in the stream; used for error reporting 569 For example: if the value of root is "config" and the stream contains this text: 570 "config.myField = 5" then this config's field "myField" is set to 5. 572 @deprecated For purposes of backwards compatibility, older config files that use 573 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 574 This feature will be removed at some point. 579 exec(stream, {}, local)
580 except NameError
as e:
581 if root ==
"config" and "root" in e.args[0]:
585 filename = getattr(stream,
"co_filename",
None)
587 filename = getattr(stream,
"name",
"?")
588 sys.stderr.write(
u"Config override file %r" % (filename,) +
589 u" appears to use 'root' instead of 'config'; trying with 'root'")
590 local = {
"root": self}
591 exec(stream, {}, local)
595 self._imports.
update(importer.getModules())
597 def save(self, filename, root="config"):
598 """!Save a python script to the named file, which, when loaded, reproduces this Config 600 @param[in] filename name of file to which to write the config 601 @param[in] root name to use for the root config variable; the same value must be used when loading 603 d = os.path.dirname(filename)
604 with tempfile.NamedTemporaryFile(mode=
"w", delete=
False, dir=d)
as outfile:
609 umask = os.umask(0o077)
611 os.chmod(outfile.name, (~umask & 0o666))
615 shutil.move(outfile.name, filename)
618 """!Save a python script to a stream, which, when loaded, reproduces this Config 620 @param outfile [inout] open file object to which to write the config. Accepts strings not bytes. 621 @param root [in] name to use for the root config variable; the same value must be used when loading 626 configType = type(self)
627 typeString = _typeStr(configType)
628 outfile.write(
u"import {}\n".
format(configType.__module__))
629 outfile.write(
u"assert type({})=={}, 'config is of type %s.%s ".
format(root, typeString))
630 outfile.write(
u"instead of {}' % (type({}).__module__, type({}).__name__)\n".
format(typeString,
638 """!Make this Config and all sub-configs read-only 644 def _save(self, outfile):
645 """!Save this Config to an open stream object 647 for imp
in self._imports:
648 if imp
in sys.modules
and sys.modules[imp]
is not None:
649 outfile.write(
u"import {}\n".
format(imp))
651 field.save(outfile, self)
654 """!Return a dict of field name: value 656 Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new 657 Field type, you may need to implement your own toDict method. 661 dict_[name] = field.toDict(self)
665 """!Return all the keys in a config recursively 671 with io.StringIO()
as strFd:
673 contents = strFd.getvalue()
679 for line
in contents.split(
"\n"):
680 if re.search(
r"^((assert|import)\s+|\s*$|#)", line):
683 mat = re.search(
r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
685 keys.append(mat.group(1))
689 def _rename(self, name):
690 """!Rename this Config object in its parent config 692 @param[in] name new name for this config in its parent config 694 Correct behavior is dependent on proper implementation of Field.rename. If implementing a new 695 Field type, you may need to implement your own rename method. 702 """!Validate the Config; raise an exception if invalid 704 The base class implementation performs type checks on all fields by 705 calling Field.validate(). 707 Complex single-field validation can be defined by deriving new Field 708 types. As syntactic sugar, some derived Field types are defined in 709 this module which handle recursing into sub-configs 710 (ConfigField, ConfigChoiceField) 712 Inter-field relationships should only be checked in derived Config 713 classes after calling this method, and base validation is complete 719 """!Format the specified config field's history to a more human-readable format 721 @param[in] name name of field whose history is wanted 722 @param[in] kwargs keyword arguments for lsst.pex.config.history.format 723 @return a string containing the formatted history 726 return pexHist.format(self, name, **kwargs)
729 Read-only history property 731 history = property(
lambda x: x._history)
734 """!Regulate which attributes can be set 736 Unlike normal python objects, Config objects are locked such 737 that no additional attributes nor properties may be added to them 740 Although this is not the standard Python behavior, it helps to 741 protect users from accidentally mispelling a field name, or 742 trying to set a non-existent field. 748 self.
_fields[attr].__set__(self, value, at=at, label=label)
749 elif hasattr(getattr(self.__class__, attr,
None),
'__set__'):
751 return object.__setattr__(self, attr, value)
752 elif attr
in self.__dict__
or attr
in (
"_name",
"_history",
"_storage",
"_frozen",
"_imports"):
754 self.__dict__[attr] = value
757 raise AttributeError(
"%s has no attribute %s" % (_typeStr(self), attr))
763 self.
_fields[attr].__delete__(self, at=at, label=label)
765 object.__delattr__(self, attr)
768 if type(other) == type(self):
770 thisValue = getattr(self, name)
771 otherValue = getattr(other, name)
772 if isinstance(thisValue, float)
and math.isnan(thisValue):
773 if not math.isnan(otherValue):
775 elif thisValue != otherValue:
781 return not self.
__eq__(other)
789 ", ".join(
"%s=%r" % (k, v)
for k, v
in self.
toDict().
items()
if v
is not None)
792 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
793 """!Compare two Configs for equality; return True if equal 795 If the Configs contain RegistryFields or ConfigChoiceFields, unselected Configs 796 will not be compared. 798 @param[in] other Config object to compare with self. 799 @param[in] shortcut If True, return as soon as an inequality is found. 800 @param[in] rtol Relative tolerance for floating point comparisons. 801 @param[in] atol Absolute tolerance for floating point comparisons. 802 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 803 to report inequalities. 805 Floating point comparisons are performed by numpy.allclose; refer to that for details. 807 name1 = self.
_name if self.
_name is not None else "config" 808 name2 = other._name
if other._name
is not None else "config" 811 rtol=rtol, atol=atol, output=output)
816 config.loadFromStream(stream)
def toDict(self, instance)
def formatHistory(self, name, kwargs)
Format the specified config field's history to a more human-readable format.
def load(self, filename, root="config")
Modify this config in place by executing the Python code in the named file.
def _rename(self, name)
Rename this Config object in its parent config.
def __init__(self, doc, dtype, default=None, check=None, optional=False)
def compareConfigs(name, c1, c2, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
def unreduceConfig(cls, stream)
def _save(self, outfile)
Save this Config to an open stream object.
def __init__(self, field, config, msg)
def saveToStream(self, outfile, root="config")
Save a python script to a stream, which, when loaded, reproduces this Config.
def __delattr__(self, attr, at=None, label="deletion")
def __get__(self, instance, owner=None, at=None, label="default")
def save(self, filename, root="config")
Save a python script to the named file, which, when loaded, reproduces this Config.
def iteritems(self)
Iterate over (field name, field value) pairs.
def _setup(self, doc, dtype, default, check, optional, source)
def __iter__(self)
Iterate over fields.
def values(self)
Return the list of field values.
def getStackFrame(relative=0)
def __contains__(self, name)
Return True if the specified field exists in this config.
def save(self, outfile, instance)
def _validateValue(self, value)
def validate(self, instance)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def iterkeys(self)
Iterate over field names.
def __new__(cls, args, kw)
Allocate a new Config object.
def validate(self)
Validate the Config; raise an exception if invalid.
def __setattr__(self, attr, value, at=None, label="assignment")
Regulate which attributes can be set.
def keys(self)
Return the list of field names.
def freeze(self, instance)
def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None)
def names(self)
Return all the keys in a config recursively.
def itervalues(self)
Iterate over field values.
def update(self, kw)
Update values specified by the keyword arguments.
def __set__(self, instance, value, at=None, label='assignment')
def items(self)
Return the list of (field name, field value) pairs.
def loadFromStream(self, stream, root="config", filename=None)
Modify this config in place by executing the python code in the provided stream.
def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Compare two Configs for equality; return True if equal.
def toDict(self)
Return a dict of field name: value.
def getComparisonName(name1, name2)
def rename(self, instance)
def __delete__(self, instance, at=None, label='deletion')
def find_module(self, fullname, path=None)
def freeze(self)
Make this Config and all sub-configs read-only.