22 __all__ = (
"Config",
"Field",
"FieldValidationError")
33 from .comparison
import getComparisonName, compareScalars, compareConfigs
34 from .callStack
import getStackFrame, getCallStack
37 def _joinNamePath(prefix=None, name=None, index=None):
39 Utility function for generating nested configuration names 41 if not prefix
and not name:
42 raise ValueError(
"Invalid name: cannot be None")
46 name = prefix +
"." + name
49 return "%s[%r]" % (name, index)
54 def _autocast(x, dtype):
56 If appropriate perform type casting of value x to type dtype, 57 otherwise return the original value x 59 if dtype == float
and isinstance(x, int):
66 Utility function to generate a fully qualified type name. 68 This is used primarily in writing config files to be 69 executed later upon 'load'. 71 if hasattr(x,
'__module__')
and hasattr(x,
'__name__'):
75 if (sys.version_info.major <= 2
and xtype.__module__ ==
'__builtin__')
or xtype.__module__ ==
'builtins':
78 return "%s.%s" % (xtype.__module__, xtype.__name__)
82 """A metaclass for Config 84 Adds a dictionary containing all Field class attributes 85 as a class attribute called '_fields', and adds the name of each field as 86 an instance variable of the field itself (so you don't have to pass the 87 name of the field to the field constructor). 90 type.__init__(cls, name, bases, dict_)
94 def getFields(classtype):
96 bases = list(classtype.__bases__)
99 fields.update(getFields(b))
101 for k, v
in classtype.__dict__.items():
102 if isinstance(v, Field):
106 fields = getFields(cls)
107 for k, v
in fields.items():
108 setattr(cls, k, copy.deepcopy(v))
111 if isinstance(value, Field):
114 type.__setattr__(cls, name, value)
119 Custom exception class which holds additional information useful to 120 debuggin Config errors: 121 - fieldType: type of the Field which incurred the error 122 - fieldName: name of the Field which incurred the error 123 - fullname: fully qualified name of the Field instance 124 - history: full history of all changes to the Field instance 125 - fieldSource: file and line number of the Field definition 130 self.
fullname = _joinNamePath(config._name, field.name)
131 self.
history = config.history.setdefault(field.name, [])
134 error =
"%s '%s' failed validation: %s\n"\
135 "For more information read the Field definition at:\n%s"\
136 "And the Config definition at:\n%s" % \
139 ValueError.__init__(self, error)
143 """A field in a a Config. 145 Instances of Field should be class attributes of Config subclasses: 146 Field only supports basic data types (int, float, complex, bool, str) 148 class Example(Config): 149 myInt = Field(int, "an integer field!", default=0) 153 supportedTypes = set((str, bool, float, int, complex))
155 def __init__(self, doc, dtype, default=None, check=None, optional=False):
156 """Initialize a Field. 158 dtype ------ Data type for the field. 159 doc -------- Documentation for the field. 160 default ---- A default value for the field. 161 check ------ A callable to be called with the field value that returns 162 False if the value is invalid. More complex inter-field 163 validation can be written as part of Config validate() 164 method; this will be ignored if set to None. 165 optional --- When False, Config validate() will fail if value is None 168 raise ValueError(
"Unsupported Field dtype %s" % _typeStr(dtype))
171 self.
_setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
173 def _setup(self, doc, dtype, default, check, optional, source):
175 Convenience function provided to simplify initialization of derived 180 self.
__doc__ = f
"{doc} (`{dtype.__name__}`" 181 if optional
or default
is not None:
182 self.
__doc__ += f
", default ``{default!r}``" 191 Rename an instance of this field, not the field itself. 192 This is invoked by the owning config object and should not be called 195 Only useful for fields which hold sub-configs. 196 Fields which hold subconfigs should rename each sub-config with 197 the full field name as generated by _joinNamePath 203 Base validation for any field. 204 Ensures that non-optional fields are not None. 205 Ensures type correctness 206 Ensures that user-provided check function is valid 207 Most derived Field types should call Field.validate if they choose 208 to re-implement validate 210 value = self.__get__(instance)
211 if not self.optional
and value
is None:
212 raise FieldValidationError(self, instance,
"Required value cannot be None")
216 Make this field read-only. 217 Only important for fields which hold sub-configs. 218 Fields which hold subconfigs should freeze each sub-config. 222 def _validateValue(self, value):
224 Validate a value that is not None 226 This is called from __set__ 227 This is not part of the Field API. However, simple derived field types 228 may benefit from implementing _validateValue 233 if not isinstance(value, self.dtype):
234 msg =
"Value %s is of incorrect type %s. Expected type %s" % \
235 (value, _typeStr(value), _typeStr(self.dtype))
237 if self.check
is not None and not self.check(value):
238 msg =
"Value %s is not a valid value" % str(value)
239 raise ValueError(msg)
241 def save(self, outfile, instance):
243 Saves an instance of this field to file. 244 This is invoked by the owning config object, and should not be called 247 outfile ---- an open output stream. 250 fullname = _joinNamePath(instance._name, self.name)
253 doc =
"# " + str(self.
doc).replace(
"\n",
"\n# ")
254 if isinstance(value, float)
and (math.isinf(value)
or math.isnan(value)):
256 outfile.write(
u"{}\n{}=float('{!r}')\n\n".
format(doc, fullname, value))
258 outfile.write(
u"{}\n{}={!r}\n\n".
format(doc, fullname, value))
262 Convert the field value so that it can be set as the value of an item 264 This is invoked by the owning config object and should not be called 267 Simple values are passed through. Complex data structures must be 268 manipulated. For example, a field holding a sub-config should, instead 269 of the subconfig object, return a dict where the keys are the field 270 names in the subconfig, and the values are the field values in the 275 def __get__(self, instance, owner=None, at=None, label="default"):
277 Define how attribute access should occur on the Config instance 278 This is invoked by the owning config object and should not be called 281 When the field attribute is accessed on a Config class object, it 282 returns the field object itself in order to allow inspection of 285 When the field attribute is access on a config instance, the actual 286 value described by the field (and held by the Config instance) is 289 if instance
is None or not isinstance(instance, Config):
292 return instance._storage[self.name]
294 def __set__(self, instance, value, at=None, label='assignment'):
296 Describe how attribute setting should occur on the config instance. 297 This is invoked by the owning config object and should not be called 300 Derived Field classes may need to override the behavior. When overriding 301 __set__, Field authors should follow the following rules: 302 * Do not allow modification of frozen configs 303 * Validate the new value *BEFORE* modifying the field. Except if the 304 new value is None. None is special and no attempt should be made to 305 validate it until Config.validate is called. 306 * Do not modify the Config instance to contain invalid values. 307 * If the field is modified, update the history of the field to reflect the 310 In order to decrease the need to implement this method in derived Field 311 types, value validation is performed in the method _validateValue. If 312 only the validation step differs in the derived Field, it is simpler to 313 implement _validateValue than to re-implement __set__. More complicated 314 behavior, however, may require a reimplementation. 319 history = instance._history.setdefault(self.name, [])
320 if value
is not None:
321 value = _autocast(value, self.
dtype)
324 except BaseException
as e:
327 instance._storage[self.name] = value
330 history.append((value, at, label))
334 Describe how attribute deletion should occur on the Config instance. 335 This is invoked by the owning config object and should not be called 340 self.
__set__(instance,
None, at=at, label=label)
342 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
343 """Helper function for Config.compare; used to compare two fields for equality. 345 Must be overridden by more complex field types. 347 @param[in] instance1 LHS Config instance to compare. 348 @param[in] instance2 RHS Config instance to compare. 349 @param[in] shortcut If True, return as soon as an inequality is found. 350 @param[in] rtol Relative tolerance for floating point comparisons. 351 @param[in] atol Absolute tolerance for floating point comparisons. 352 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 353 to report inequalities. 355 Floating point comparisons are performed by numpy.allclose; refer to that for details. 357 v1 = getattr(instance1, self.name)
358 v2 = getattr(instance2, self.name)
360 _joinNamePath(instance1._name, self.name),
361 _joinNamePath(instance2._name, self.name)
367 """An Importer (for sys.meta_path) that records which modules are being imported. 369 Objects also act as Context Managers, so you can: 370 with RecordingImporter() as importer: 372 print("Imported: " + importer.getModules()) 373 This ensures it is properly uninstalled when done. 375 This class makes no effort to do any importing itself. 378 """Create and install the Importer""" 384 sys.meta_path = [self] + sys.meta_path
392 """Uninstall the Importer""" 396 """Called as part of the 'import' chain of events. 398 We return None because we don't do any importing. 404 """Return the set of modules that were imported.""" 409 """Base class for control objects. 411 A Config object will usually have several Field instances as class 412 attributes; these are used to define most of the base class behavior. 413 Simple derived class should be able to be defined simply by setting those 416 Config also emulates a dict of field name: field value 420 """!Iterate over fields 425 """!Return the list of field names 427 return list(self._storage.
keys())
430 """!Return the list of field values 432 return list(self._storage.
values())
435 """!Return the list of (field name, field value) pairs 437 return list(self._storage.
items())
440 """!Iterate over (field name, field value) pairs 442 return iter(self._storage.
items())
445 """!Iterate over field values 447 return iter(self.storage.
values())
450 """!Iterate over field names 452 return iter(self.storage.
keys())
455 """!Return True if the specified field exists in this config 457 @param[in] name field name to test for 462 """!Allocate a new Config object. 464 In order to ensure that all Config object are always in a proper 465 state when handed to users or to derived Config classes, some 466 attributes are handled at allocation time rather than at initialization 468 This ensures that even if a derived Config class implements __init__, 469 the author does not need to be concerned about when or even if he 470 should call the base Config.__init__ 472 name = kw.pop(
"__name",
None)
475 kw.pop(
"__label",
"default")
477 instance = object.__new__(cls)
478 instance._frozen =
False 479 instance._name = name
480 instance._storage = {}
481 instance._history = {}
482 instance._imports = set()
484 for field
in instance._fields.values():
485 instance._history[field.name] = []
486 field.__set__(instance, field.default, at=at + [field.source], label=
"default")
488 instance.setDefaults()
490 instance.update(__at=at, **kw)
494 """Reduction for pickling (function with arguments to reproduce). 496 We need to condense and reconstitute the Config, since it may contain lambdas 497 (as the 'check' elements) that cannot be pickled. 500 stream = io.StringIO()
502 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
506 Derived config classes that must compute defaults rather than using the 507 Field defaults should do so here. 508 To correctly use inherited defaults, implementations of setDefaults() 509 must call their base class' setDefaults() 514 """!Update values specified by the keyword arguments 516 @warning The '__at' and '__label' keyword arguments are special internal 517 keywords. They are used to strip out any internal steps from the 518 history tracebacks of the config. Modifying these keywords allows users 519 to lie about a Config's history. Please do not do so! 522 label = kw.pop(
"__label",
"update")
524 for name, value
in kw.items():
526 field = self._fields[name]
527 field.__set__(self, value, at=at, label=label)
529 raise KeyError(
"No field of name %s exists in config type %s" % (name, _typeStr(self)))
531 def load(self, filename, root="config"):
532 """!Modify this config in place by executing the Python code in the named file. 534 @param[in] filename name of file containing config override code 535 @param[in] root name of variable in file that refers to the config being overridden 537 For example: if the value of root is "config" and the file contains this text: 538 "config.myField = 5" then this config's field "myField" is set to 5. 540 @deprecated For purposes of backwards compatibility, older config files that use 541 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 542 This feature will be removed at some point. 544 with open(filename,
"r") as f: 545 code = compile(f.read(), filename=filename, mode="exec")
549 """!Modify this config in place by executing the python code in the provided stream. 551 @param[in] stream open file object, string or compiled string containing config override code 552 @param[in] root name of variable in stream that refers to the config being overridden 553 @param[in] filename name of config override file, or None if unknown or contained 554 in the stream; used for error reporting 556 For example: if the value of root is "config" and the stream contains this text: 557 "config.myField = 5" then this config's field "myField" is set to 5. 559 @deprecated For purposes of backwards compatibility, older config files that use 560 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 561 This feature will be removed at some point. 566 exec(stream, {}, local)
567 except NameError
as e:
568 if root ==
"config" and "root" in e.args[0]:
572 filename = getattr(stream,
"co_filename",
None)
574 filename = getattr(stream,
"name",
"?")
575 print(f
"Config override file {filename!r}" 576 " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
577 local = {
"root": self}
578 exec(stream, {}, local)
582 self._imports.
update(importer.getModules())
584 def save(self, filename, root="config"):
585 """!Save a python script to the named file, which, when loaded, reproduces this Config 587 @param[in] filename name of file to which to write the config 588 @param[in] root name to use for the root config variable; the same value must be used when loading 590 d = os.path.dirname(filename)
591 with tempfile.NamedTemporaryFile(mode=
"w", delete=
False, dir=d)
as outfile:
596 umask = os.umask(0o077)
598 os.chmod(outfile.name, (~umask & 0o666))
602 shutil.move(outfile.name, filename)
605 """!Save a python script to a stream, which, when loaded, reproduces this Config 607 @param outfile [inout] open file object to which to write the config. Accepts strings not bytes. 608 @param root [in] name to use for the root config variable; the same value must be used when loading 613 configType = type(self)
614 typeString = _typeStr(configType)
615 outfile.write(
u"import {}\n".
format(configType.__module__))
616 outfile.write(
u"assert type({})=={}, 'config is of type %s.%s ".
format(root, typeString))
617 outfile.write(
u"instead of {}' % (type({}).__module__, type({}).__name__)\n".
format(typeString,
625 """!Make this Config and all sub-configs read-only 631 def _save(self, outfile):
632 """!Save this Config to an open stream object 634 for imp
in self._imports:
635 if imp
in sys.modules
and sys.modules[imp]
is not None:
636 outfile.write(
u"import {}\n".
format(imp))
638 field.save(outfile, self)
641 """!Return a dict of field name: value 643 Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new 644 Field type, you may need to implement your own toDict method. 648 dict_[name] = field.toDict(self)
652 """!Return all the keys in a config recursively 658 with io.StringIO()
as strFd:
660 contents = strFd.getvalue()
666 for line
in contents.split(
"\n"):
667 if re.search(
r"^((assert|import)\s+|\s*$|#)", line):
670 mat = re.search(
r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
672 keys.append(mat.group(1))
676 def _rename(self, name):
677 """!Rename this Config object in its parent config 679 @param[in] name new name for this config in its parent config 681 Correct behavior is dependent on proper implementation of Field.rename. If implementing a new 682 Field type, you may need to implement your own rename method. 689 """!Validate the Config; raise an exception if invalid 691 The base class implementation performs type checks on all fields by 692 calling Field.validate(). 694 Complex single-field validation can be defined by deriving new Field 695 types. As syntactic sugar, some derived Field types are defined in 696 this module which handle recursing into sub-configs 697 (ConfigField, ConfigChoiceField) 699 Inter-field relationships should only be checked in derived Config 700 classes after calling this method, and base validation is complete 706 """!Format the specified config field's history to a more human-readable format 708 @param[in] name name of field whose history is wanted 709 @param[in] kwargs keyword arguments for lsst.pex.config.history.format 710 @return a string containing the formatted history 713 return pexHist.format(self, name, **kwargs)
716 Read-only history property 718 history = property(
lambda x: x._history)
721 """!Regulate which attributes can be set 723 Unlike normal python objects, Config objects are locked such 724 that no additional attributes nor properties may be added to them 727 Although this is not the standard Python behavior, it helps to 728 protect users from accidentally mispelling a field name, or 729 trying to set a non-existent field. 735 self.
_fields[attr].__set__(self, value, at=at, label=label)
736 elif hasattr(getattr(self.__class__, attr,
None),
'__set__'):
738 return object.__setattr__(self, attr, value)
739 elif attr
in self.__dict__
or attr
in (
"_name",
"_history",
"_storage",
"_frozen",
"_imports"):
741 self.__dict__[attr] = value
744 raise AttributeError(
"%s has no attribute %s" % (_typeStr(self), attr))
750 self.
_fields[attr].__delete__(self, at=at, label=label)
752 object.__delattr__(self, attr)
755 if type(other) == type(self):
757 thisValue = getattr(self, name)
758 otherValue = getattr(other, name)
759 if isinstance(thisValue, float)
and math.isnan(thisValue):
760 if not math.isnan(otherValue):
762 elif thisValue != otherValue:
768 return not self.
__eq__(other)
776 ", ".join(
"%s=%r" % (k, v)
for k, v
in self.
toDict().
items()
if v
is not None)
779 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
780 """!Compare two Configs for equality; return True if equal 782 If the Configs contain RegistryFields or ConfigChoiceFields, unselected Configs 783 will not be compared. 785 @param[in] other Config object to compare with self. 786 @param[in] shortcut If True, return as soon as an inequality is found. 787 @param[in] rtol Relative tolerance for floating point comparisons. 788 @param[in] atol Absolute tolerance for floating point comparisons. 789 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 790 to report inequalities. 792 Floating point comparisons are performed by numpy.allclose; refer to that for details. 794 name1 = self.
_name if self.
_name is not None else "config" 795 name2 = other._name
if other._name
is not None else "config" 798 rtol=rtol, atol=atol, output=output)
803 config.loadFromStream(stream)
def toDict(self, instance)
def formatHistory(self, name, kwargs)
Format the specified config field's history to a more human-readable format.
def load(self, filename, root="config")
Modify this config in place by executing the Python code in the named file.
def _rename(self, name)
Rename this Config object in its parent config.
def __init__(self, doc, dtype, default=None, check=None, optional=False)
def compareConfigs(name, c1, c2, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
def unreduceConfig(cls, stream)
def _save(self, outfile)
Save this Config to an open stream object.
def __init__(self, field, config, msg)
def saveToStream(self, outfile, root="config")
Save a python script to a stream, which, when loaded, reproduces this Config.
def __delattr__(self, attr, at=None, label="deletion")
def __get__(self, instance, owner=None, at=None, label="default")
def save(self, filename, root="config")
Save a python script to the named file, which, when loaded, reproduces this Config.
def iteritems(self)
Iterate over (field name, field value) pairs.
def _setup(self, doc, dtype, default, check, optional, source)
def __iter__(self)
Iterate over fields.
def values(self)
Return the list of field values.
def getStackFrame(relative=0)
def __contains__(self, name)
Return True if the specified field exists in this config.
def save(self, outfile, instance)
def _validateValue(self, value)
def validate(self, instance)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def iterkeys(self)
Iterate over field names.
def __new__(cls, args, kw)
Allocate a new Config object.
def validate(self)
Validate the Config; raise an exception if invalid.
def __setattr__(self, attr, value, at=None, label="assignment")
Regulate which attributes can be set.
def keys(self)
Return the list of field names.
def freeze(self, instance)
def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None)
def names(self)
Return all the keys in a config recursively.
def itervalues(self)
Iterate over field values.
def update(self, kw)
Update values specified by the keyword arguments.
def __set__(self, instance, value, at=None, label='assignment')
def items(self)
Return the list of (field name, field value) pairs.
def loadFromStream(self, stream, root="config", filename=None)
Modify this config in place by executing the python code in the provided stream.
def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Compare two Configs for equality; return True if equal.
def toDict(self)
Return a dict of field name: value.
def getComparisonName(name1, name2)
def rename(self, instance)
def __delete__(self, instance, at=None, label='deletion')
def find_module(self, fullname, path=None)
def freeze(self)
Make this Config and all sub-configs read-only.