Coverage for python/lsst/pex/config/config.py: 59%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28__all__ = ("Config", "ConfigMeta", "Field", "FieldValidationError")
30import io
31import importlib
32import os
33import re
34import sys
35import math
36import copy
37import tempfile
38import shutil
39import warnings
41# if YAML is not available that's fine and we simply don't register
42# the yaml representer since we know it won't be used.
43try:
44 import yaml
45except ImportError:
46 yaml = None
47 YamlLoaders = ()
48 doImport = None
50from .comparison import getComparisonName, compareScalars, compareConfigs
51from .callStack import getStackFrame, getCallStack
53if yaml: 53 ↛ 64line 53 didn't jump to line 64, because the condition on line 53 was never false
54 YamlLoaders = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
56 try:
57 # CLoader is not always available
58 from yaml import CLoader
59 YamlLoaders += (CLoader,)
60 except ImportError:
61 pass
64def _joinNamePath(prefix=None, name=None, index=None):
65 """Generate nested configuration names.
66 """
67 if not prefix and not name: 67 ↛ 68line 67 didn't jump to line 68, because the condition on line 67 was never true
68 raise ValueError("Invalid name: cannot be None")
69 elif not name: 69 ↛ 70line 69 didn't jump to line 70, because the condition on line 69 was never true
70 name = prefix
71 elif prefix and name: 71 ↛ 74line 71 didn't jump to line 74, because the condition on line 71 was never false
72 name = prefix + "." + name
74 if index is not None: 74 ↛ 75line 74 didn't jump to line 75, because the condition on line 74 was never true
75 return "%s[%r]" % (name, index)
76 else:
77 return name
80def _autocast(x, dtype):
81 """Cast a value to a type, if appropriate.
83 Parameters
84 ----------
85 x : object
86 A value.
87 dtype : tpye
88 Data type, such as `float`, `int`, or `str`.
90 Returns
91 -------
92 values : object
93 If appropriate, the returned value is ``x`` cast to the given type
94 ``dtype``. If the cast cannot be performed the original value of
95 ``x`` is returned.
96 """
97 if dtype == float and isinstance(x, int):
98 return float(x)
99 return x
102def _typeStr(x):
103 """Generate a fully-qualified type name.
105 Returns
106 -------
107 `str`
108 Fully-qualified type name.
110 Notes
111 -----
112 This function is used primarily for writing config files to be executed
113 later upon with the 'load' function.
114 """
115 if hasattr(x, '__module__') and hasattr(x, '__name__'):
116 xtype = x
117 else:
118 xtype = type(x)
119 if (sys.version_info.major <= 2 and xtype.__module__ == '__builtin__') or xtype.__module__ == 'builtins': 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true
120 return xtype.__name__
121 else:
122 return "%s.%s" % (xtype.__module__, xtype.__name__)
125if yaml: 125 ↛ 157line 125 didn't jump to line 157, because the condition on line 125 was never false
126 def _yaml_config_representer(dumper, data):
127 """Represent a Config object in a form suitable for YAML.
129 Stores the serialized stream as a scalar block string.
130 """
131 stream = io.StringIO()
132 data.saveToStream(stream)
133 config_py = stream.getvalue()
135 # Strip multiple newlines from the end of the config
136 # This simplifies the YAML to use | and not |+
137 config_py = config_py.rstrip() + "\n"
139 # Trailing spaces force pyyaml to use non-block form.
140 # Remove the trailing spaces so it has no choice
141 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
143 # Store the Python as a simple scalar
144 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
146 def _yaml_config_constructor(loader, node):
147 """Construct a config from YAML"""
148 config_py = loader.construct_scalar(node)
149 return Config._fromPython(config_py)
151 # Register a generic constructor for Config and all subclasses
152 # Need to register for all the loaders we would like to use
153 for loader in YamlLoaders:
154 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
157class ConfigMeta(type):
158 """A metaclass for `lsst.pex.config.Config`.
160 Notes
161 -----
162 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
163 class attributes as a class attribute called ``_fields``, and adds
164 the name of each field as an instance variable of the field itself (so you
165 don't have to pass the name of the field to the field constructor).
166 """
168 def __init__(cls, name, bases, dict_):
169 type.__init__(cls, name, bases, dict_)
170 cls._fields = {}
171 cls._source = getStackFrame()
173 def getFields(classtype):
174 fields = {}
175 bases = list(classtype.__bases__)
176 bases.reverse()
177 for b in bases:
178 fields.update(getFields(b))
180 for k, v in classtype.__dict__.items():
181 if isinstance(v, Field):
182 fields[k] = v
183 return fields
185 fields = getFields(cls)
186 for k, v in fields.items():
187 setattr(cls, k, copy.deepcopy(v))
189 def __setattr__(cls, name, value):
190 if isinstance(value, Field):
191 value.name = name
192 cls._fields[name] = value
193 type.__setattr__(cls, name, value)
196class FieldValidationError(ValueError):
197 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
198 particular ``~lsst.pex.config.Config``.
200 Parameters
201 ----------
202 field : `lsst.pex.config.Field`
203 The field that was not valid.
204 config : `lsst.pex.config.Config`
205 The config containing the invalid field.
206 msg : `str`
207 Text describing why the field was not valid.
208 """
210 def __init__(self, field, config, msg):
211 self.fieldType = type(field)
212 """Type of the `~lsst.pex.config.Field` that incurred the error.
213 """
215 self.fieldName = field.name
216 """Name of the `~lsst.pex.config.Field` instance that incurred the
217 error (`str`).
219 See also
220 --------
221 lsst.pex.config.Field.name
222 """
224 self.fullname = _joinNamePath(config._name, field.name)
225 """Fully-qualified name of the `~lsst.pex.config.Field` instance
226 (`str`).
227 """
229 self.history = config.history.setdefault(field.name, [])
230 """Full history of all changes to the `~lsst.pex.config.Field`
231 instance.
232 """
234 self.fieldSource = field.source
235 """File and line number of the `~lsst.pex.config.Field` definition.
236 """
238 self.configSource = config._source
239 error = "%s '%s' failed validation: %s\n"\
240 "For more information see the Field definition at:\n%s"\
241 " and the Config definition at:\n%s" % \
242 (self.fieldType.__name__, self.fullname, msg,
243 self.fieldSource.format(), self.configSource.format())
244 super().__init__(error)
247class Field:
248 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
249 `complex`, `bool`, and `str` data types.
251 Parameters
252 ----------
253 doc : `str`
254 A description of the field for users.
255 dtype : type
256 The field's data type. ``Field`` only supports basic data types:
257 `int`, `float`, `complex`, `bool`, and `str`. See
258 `Field.supportedTypes`.
259 default : object, optional
260 The field's default value.
261 check : callable, optional
262 A callable that is called with the field's value. This callable should
263 return `False` if the value is invalid. More complex inter-field
264 validation can be written as part of the
265 `lsst.pex.config.Config.validate` method.
266 optional : `bool`, optional
267 This sets whether the field is considered optional, and therefore
268 doesn't need to be set by the user. When `False`,
269 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
270 deprecated : None or `str`, optional
271 A description of why this Field is deprecated, including removal date.
272 If not None, the string is appended to the docstring for this Field.
274 Raises
275 ------
276 ValueError
277 Raised when the ``dtype`` parameter is not one of the supported types
278 (see `Field.supportedTypes`).
280 See also
281 --------
282 ChoiceField
283 ConfigChoiceField
284 ConfigDictField
285 ConfigField
286 ConfigurableField
287 DictField
288 ListField
289 RangeField
290 RegistryField
292 Notes
293 -----
294 ``Field`` instances (including those of any subclass of ``Field``) are used
295 as class attributes of `~lsst.pex.config.Config` subclasses (see the
296 example, below). ``Field`` attributes work like the `property` attributes
297 of classes that implement custom setters and getters. `Field` attributes
298 belong to the class, but operate on the instance. Formally speaking,
299 `Field` attributes are `descriptors
300 <https://docs.python.org/3/howto/descriptor.html>`_.
302 When you access a `Field` attribute on a `Config` instance, you don't
303 get the `Field` instance itself. Instead, you get the value of that field,
304 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
305 container type (like a `lsst.pex.config.List`) depending on the field's
306 type. See the example, below.
308 Examples
309 --------
310 Instances of ``Field`` should be used as class attributes of
311 `lsst.pex.config.Config` subclasses:
313 >>> from lsst.pex.config import Config, Field
314 >>> class Example(Config):
315 ... myInt = Field("An integer field.", int, default=0)
316 ...
317 >>> print(config.myInt)
318 0
319 >>> config.myInt = 5
320 >>> print(config.myInt)
321 5
322 """
324 supportedTypes = set((str, bool, float, int, complex))
325 """Supported data types for field values (`set` of types).
326 """
328 def __init__(self, doc, dtype, default=None, check=None, optional=False, deprecated=None):
329 if dtype not in self.supportedTypes: 329 ↛ 330line 329 didn't jump to line 330, because the condition on line 329 was never true
330 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
332 source = getStackFrame()
333 self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source,
334 deprecated=deprecated)
336 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
337 """Set attributes, usually during initialization.
338 """
339 self.dtype = dtype
340 """Data type for the field.
341 """
343 # append the deprecation message to the docstring.
344 if deprecated is not None:
345 doc = f"{doc} Deprecated: {deprecated}"
346 self.doc = doc
347 """A description of the field (`str`).
348 """
350 self.deprecated = deprecated
351 """If not None, a description of why this field is deprecated (`str`).
352 """
354 self.__doc__ = f"{doc} (`{dtype.__name__}`"
355 if optional or default is not None:
356 self.__doc__ += f", default ``{default!r}``"
357 self.__doc__ += ")"
359 self.default = default
360 """Default value for this field.
361 """
363 self.check = check
364 """A user-defined function that validates the value of the field.
365 """
367 self.optional = optional
368 """Flag that determines if the field is required to be set (`bool`).
370 When `False`, `lsst.pex.config.Config.validate` will fail if the
371 field's value is `None`.
372 """
374 self.source = source
375 """The stack frame where this field is defined (`list` of
376 `lsst.pex.config.callStack.StackFrame`).
377 """
379 def rename(self, instance):
380 """Rename the field in a `~lsst.pex.config.Config` (for internal use
381 only).
383 Parameters
384 ----------
385 instance : `lsst.pex.config.Config`
386 The config instance that contains this field.
388 Notes
389 -----
390 This method is invoked by the `lsst.pex.config.Config` object that
391 contains this field and should not be called directly.
393 Renaming is only relevant for `~lsst.pex.config.Field` instances that
394 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should
395 rename each subconfig with the full field name as generated by
396 `lsst.pex.config.config._joinNamePath`.
397 """
398 pass
400 def validate(self, instance):
401 """Validate the field (for internal use only).
403 Parameters
404 ----------
405 instance : `lsst.pex.config.Config`
406 The config instance that contains this field.
408 Raises
409 ------
410 lsst.pex.config.FieldValidationError
411 Raised if verification fails.
413 Notes
414 -----
415 This method provides basic validation:
417 - Ensures that the value is not `None` if the field is not optional.
418 - Ensures type correctness.
419 - Ensures that the user-provided ``check`` function is valid.
421 Most `~lsst.pex.config.Field` subclasses should call
422 `lsst.pex.config.field.Field.validate` if they re-implement
423 `~lsst.pex.config.field.Field.validate`.
424 """
425 value = self.__get__(instance)
426 if not self.optional and value is None:
427 raise FieldValidationError(self, instance, "Required value cannot be None")
429 def freeze(self, instance):
430 """Make this field read-only (for internal use only).
432 Parameters
433 ----------
434 instance : `lsst.pex.config.Config`
435 The config instance that contains this field.
437 Notes
438 -----
439 Freezing is only relevant for fields that hold subconfigs. Fields which
440 hold subconfigs should freeze each subconfig.
442 **Subclasses should implement this method.**
443 """
444 pass
446 def _validateValue(self, value):
447 """Validate a value.
449 Parameters
450 ----------
451 value : object
452 The value being validated.
454 Raises
455 ------
456 TypeError
457 Raised if the value's type is incompatible with the field's
458 ``dtype``.
459 ValueError
460 Raised if the value is rejected by the ``check`` method.
461 """
462 if value is None: 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true
463 return
465 if not isinstance(value, self.dtype): 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true
466 msg = "Value %s is of incorrect type %s. Expected type %s" % \
467 (value, _typeStr(value), _typeStr(self.dtype))
468 raise TypeError(msg)
469 if self.check is not None and not self.check(value): 469 ↛ 470line 469 didn't jump to line 470, because the condition on line 469 was never true
470 msg = "Value %s is not a valid value" % str(value)
471 raise ValueError(msg)
473 def _collectImports(self, instance, imports):
474 """This function should call the _collectImports method on all config
475 objects the field may own, and union them with the supplied imports
476 set.
478 Parameters
479 ----------
480 instance : instance or subclass of `lsst.pex.config.Config`
481 A config object that has this field defined on it
482 imports : `set`
483 Set of python modules that need imported after persistence
484 """
485 pass
487 def save(self, outfile, instance):
488 """Save this field to a file (for internal use only).
490 Parameters
491 ----------
492 outfile : file-like object
493 A writeable field handle.
494 instance : `Config`
495 The `Config` instance that contains this field.
497 Notes
498 -----
499 This method is invoked by the `~lsst.pex.config.Config` object that
500 contains this field and should not be called directly.
502 The output consists of the documentation string
503 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
504 line is formatted as an assignment: ``{fullname}={value}``.
506 This output can be executed with Python.
507 """
508 value = self.__get__(instance)
509 fullname = _joinNamePath(instance._name, self.name)
511 if self.deprecated and value == self.default: 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true
512 return
514 # write full documentation string as comment lines
515 # (i.e. first character is #)
516 doc = "# " + str(self.doc).replace("\n", "\n# ")
517 if isinstance(value, float) and not math.isfinite(value): 517 ↛ 519line 517 didn't jump to line 519, because the condition on line 517 was never true
518 # non-finite numbers need special care
519 outfile.write(u"{}\n{}=float('{!r}')\n\n".format(doc, fullname, value))
520 else:
521 outfile.write(u"{}\n{}={!r}\n\n".format(doc, fullname, value))
523 def toDict(self, instance):
524 """Convert the field value so that it can be set as the value of an
525 item in a `dict` (for internal use only).
527 Parameters
528 ----------
529 instance : `Config`
530 The `Config` that contains this field.
532 Returns
533 -------
534 value : object
535 The field's value. See *Notes*.
537 Notes
538 -----
539 This method invoked by the owning `~lsst.pex.config.Config` object and
540 should not be called directly.
542 Simple values are passed through. Complex data structures must be
543 manipulated. For example, a `~lsst.pex.config.Field` holding a
544 subconfig should, instead of the subconfig object, return a `dict`
545 where the keys are the field names in the subconfig, and the values are
546 the field values in the subconfig.
547 """
548 return self.__get__(instance)
550 def __get__(self, instance, owner=None, at=None, label="default"):
551 """Define how attribute access should occur on the Config instance
552 This is invoked by the owning config object and should not be called
553 directly
555 When the field attribute is accessed on a Config class object, it
556 returns the field object itself in order to allow inspection of
557 Config classes.
559 When the field attribute is access on a config instance, the actual
560 value described by the field (and held by the Config instance) is
561 returned.
562 """
563 if instance is None: 563 ↛ 564line 563 didn't jump to line 564, because the condition on line 563 was never true
564 return self
565 else:
566 # try statements are almost free in python if they succeed
567 try:
568 return instance._storage[self.name]
569 except AttributeError:
570 if not isinstance(instance, Config):
571 return self
572 else:
573 raise AttributeError(f"Config {instance} is missing "
574 "_storage attribute, likely"
575 " incorrectly initialized")
577 def __set__(self, instance, value, at=None, label='assignment'):
578 """Set an attribute on the config instance.
580 Parameters
581 ----------
582 instance : `lsst.pex.config.Config`
583 The config instance that contains this field.
584 value : obj
585 Value to set on this field.
586 at : `list` of `lsst.pex.config.callStack.StackFrame`
587 The call stack (created by
588 `lsst.pex.config.callStack.getCallStack`).
589 label : `str`, optional
590 Event label for the history.
592 Notes
593 -----
594 This method is invoked by the owning `lsst.pex.config.Config` object
595 and should not be called directly.
597 Derived `~lsst.pex.config.Field` classes may need to override the
598 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
599 should follow the following rules:
601 - Do not allow modification of frozen configs.
602 - Validate the new value **before** modifying the field. Except if the
603 new value is `None`. `None` is special and no attempt should be made
604 to validate it until `lsst.pex.config.Config.validate` is called.
605 - Do not modify the `~lsst.pex.config.Config` instance to contain
606 invalid values.
607 - If the field is modified, update the history of the
608 `lsst.pex.config.field.Field` to reflect the changes.
610 In order to decrease the need to implement this method in derived
611 `~lsst.pex.config.Field` types, value validation is performed in the
612 `lsst.pex.config.Field._validateValue`. If only the validation step
613 differs in the derived `~lsst.pex.config.Field`, it is simpler to
614 implement `lsst.pex.config.Field._validateValue` than to reimplement
615 ``__set__``. More complicated behavior, however, may require
616 reimplementation.
617 """
618 if instance._frozen: 618 ↛ 619line 618 didn't jump to line 619, because the condition on line 618 was never true
619 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
621 history = instance._history.setdefault(self.name, [])
622 if value is not None: 622 ↛ 629line 622 didn't jump to line 629, because the condition on line 622 was never false
623 value = _autocast(value, self.dtype)
624 try:
625 self._validateValue(value)
626 except BaseException as e:
627 raise FieldValidationError(self, instance, str(e))
629 instance._storage[self.name] = value
630 if at is None: 630 ↛ 631line 630 didn't jump to line 631, because the condition on line 630 was never true
631 at = getCallStack()
632 history.append((value, at, label))
634 def __delete__(self, instance, at=None, label='deletion'):
635 """Delete an attribute from a `lsst.pex.config.Config` instance.
637 Parameters
638 ----------
639 instance : `lsst.pex.config.Config`
640 The config instance that contains this field.
641 at : `list` of `lsst.pex.config.callStack.StackFrame`
642 The call stack (created by
643 `lsst.pex.config.callStack.getCallStack`).
644 label : `str`, optional
645 Event label for the history.
647 Notes
648 -----
649 This is invoked by the owning `~lsst.pex.config.Config` object and
650 should not be called directly.
651 """
652 if at is None:
653 at = getCallStack()
654 self.__set__(instance, None, at=at, label=label)
656 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
657 """Compare a field (named `Field.name`) in two
658 `~lsst.pex.config.Config` instances for equality.
660 Parameters
661 ----------
662 instance1 : `lsst.pex.config.Config`
663 Left-hand side `Config` instance to compare.
664 instance2 : `lsst.pex.config.Config`
665 Right-hand side `Config` instance to compare.
666 shortcut : `bool`, optional
667 **Unused.**
668 rtol : `float`, optional
669 Relative tolerance for floating point comparisons.
670 atol : `float`, optional
671 Absolute tolerance for floating point comparisons.
672 output : callable, optional
673 A callable that takes a string, used (possibly repeatedly) to
674 report inequalities.
676 Notes
677 -----
678 This method must be overridden by more complex `Field` subclasses.
680 See also
681 --------
682 lsst.pex.config.compareScalars
683 """
684 v1 = getattr(instance1, self.name)
685 v2 = getattr(instance2, self.name)
686 name = getComparisonName(
687 _joinNamePath(instance1._name, self.name),
688 _joinNamePath(instance2._name, self.name)
689 )
690 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
693class RecordingImporter:
694 """Importer (for `sys.meta_path`) that records which modules are being
695 imported.
697 *This class does not do any importing itself.*
699 Examples
700 --------
701 Use this class as a context manager to ensure it is properly uninstalled
702 when done:
704 >>> with RecordingImporter() as importer:
705 ... # import stuff
706 ... import numpy as np
707 ... print("Imported: " + importer.getModules())
708 """
710 def __init__(self):
711 self._modules = set()
713 def __enter__(self):
714 self.origMetaPath = sys.meta_path
715 sys.meta_path = [self] + sys.meta_path
716 return self
718 def __exit__(self, *args):
719 self.uninstall()
720 return False # Don't suppress exceptions
722 def uninstall(self):
723 """Uninstall the importer.
724 """
725 sys.meta_path = self.origMetaPath
727 def find_module(self, fullname, path=None):
728 """Called as part of the ``import`` chain of events.
729 """
730 self._modules.add(fullname)
731 # Return None because we don't do any importing.
732 return None
734 def getModules(self):
735 """Get the set of modules that were imported.
737 Returns
738 -------
739 modules : `set` of `str`
740 Set of imported module names.
741 """
742 return self._modules
745class Config(metaclass=ConfigMeta):
746 """Base class for configuration (*config*) objects.
748 Notes
749 -----
750 A ``Config`` object will usually have several `~lsst.pex.config.Field`
751 instances as class attributes. These are used to define most of the base
752 class behavior.
754 ``Config`` implements a mapping API that provides many `dict`-like methods,
755 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and
756 `itervalues`. ``Config`` instances also support the ``in`` operator to
757 test if a field is in the config. Unlike a `dict`, ``Config`` classes are
758 not subscriptable. Instead, access individual fields as attributes of the
759 configuration instance.
761 Examples
762 --------
763 Config classes are subclasses of ``Config`` that have
764 `~lsst.pex.config.Field` instances (or instances of
765 `~lsst.pex.config.Field` subclasses) as class attributes:
767 >>> from lsst.pex.config import Config, Field, ListField
768 >>> class DemoConfig(Config):
769 ... intField = Field(doc="An integer field", dtype=int, default=42)
770 ... listField = ListField(doc="List of favorite beverages.", dtype=str,
771 ... default=['coffee', 'green tea', 'water'])
772 ...
773 >>> config = DemoConfig()
775 Configs support many `dict`-like APIs:
777 >>> config.keys()
778 ['intField', 'listField']
779 >>> 'intField' in config
780 True
782 Individual fields can be accessed as attributes of the configuration:
784 >>> config.intField
785 42
786 >>> config.listField.append('earl grey tea')
787 >>> print(config.listField)
788 ['coffee', 'green tea', 'water', 'earl grey tea']
789 """
791 def __iter__(self):
792 """Iterate over fields.
793 """
794 return self._fields.__iter__()
796 def keys(self):
797 """Get field names.
799 Returns
800 -------
801 names : `list`
802 List of `lsst.pex.config.Field` names.
804 See also
805 --------
806 lsst.pex.config.Config.iterkeys
807 """
808 return list(self._storage.keys())
810 def values(self):
811 """Get field values.
813 Returns
814 -------
815 values : `list`
816 List of field values.
818 See also
819 --------
820 lsst.pex.config.Config.itervalues
821 """
822 return list(self._storage.values())
824 def items(self):
825 """Get configurations as ``(field name, field value)`` pairs.
827 Returns
828 -------
829 items : `list`
830 List of tuples for each configuration. Tuple items are:
832 0. Field name.
833 1. Field value.
835 See also
836 --------
837 lsst.pex.config.Config.iteritems
838 """
839 return list(self._storage.items())
841 def iteritems(self):
842 """Iterate over (field name, field value) pairs.
844 Yields
845 ------
846 item : `tuple`
847 Tuple items are:
849 0. Field name.
850 1. Field value.
852 See also
853 --------
854 lsst.pex.config.Config.items
855 """
856 return iter(self._storage.items())
858 def itervalues(self):
859 """Iterate over field values.
861 Yields
862 ------
863 value : obj
864 A field value.
866 See also
867 --------
868 lsst.pex.config.Config.values
869 """
870 return iter(self.storage.values())
872 def iterkeys(self):
873 """Iterate over field names
875 Yields
876 ------
877 key : `str`
878 A field's key (attribute name).
880 See also
881 --------
882 lsst.pex.config.Config.values
883 """
884 return iter(self.storage.keys())
886 def __contains__(self, name):
887 """!Return True if the specified field exists in this config
889 @param[in] name field name to test for
890 """
891 return self._storage.__contains__(name)
893 def __new__(cls, *args, **kw):
894 """Allocate a new `lsst.pex.config.Config` object.
896 In order to ensure that all Config object are always in a proper state
897 when handed to users or to derived `~lsst.pex.config.Config` classes,
898 some attributes are handled at allocation time rather than at
899 initialization.
901 This ensures that even if a derived `~lsst.pex.config.Config` class
902 implements ``__init__``, its author does not need to be concerned about
903 when or even the base ``Config.__init__`` should be called.
904 """
905 name = kw.pop("__name", None)
906 at = kw.pop("__at", getCallStack())
907 # remove __label and ignore it
908 kw.pop("__label", "default")
910 instance = object.__new__(cls)
911 instance._frozen = False
912 instance._name = name
913 instance._storage = {}
914 instance._history = {}
915 instance._imports = set()
916 # load up defaults
917 for field in instance._fields.values():
918 instance._history[field.name] = []
919 field.__set__(instance, field.default, at=at + [field.source], label="default")
920 # set custom default-overides
921 instance.setDefaults()
922 # set constructor overides
923 instance.update(__at=at, **kw)
924 return instance
926 def __reduce__(self):
927 """Reduction for pickling (function with arguments to reproduce).
929 We need to condense and reconstitute the `~lsst.pex.config.Config`,
930 since it may contain lambdas (as the ``check`` elements) that cannot
931 be pickled.
932 """
933 # The stream must be in characters to match the API but pickle
934 # requires bytes
935 stream = io.StringIO()
936 self.saveToStream(stream)
937 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
939 def setDefaults(self):
940 """Subclass hook for computing defaults.
942 Notes
943 -----
944 Derived `~lsst.pex.config.Config` classes that must compute defaults
945 rather than using the `~lsst.pex.config.Field` instances's defaults
946 should do so here. To correctly use inherited defaults,
947 implementations of ``setDefaults`` must call their base class's
948 ``setDefaults``.
949 """
950 pass
952 def update(self, **kw):
953 """Update values of fields specified by the keyword arguments.
955 Parameters
956 ----------
957 kw
958 Keywords are configuration field names. Values are configuration
959 field values.
961 Notes
962 -----
963 The ``__at`` and ``__label`` keyword arguments are special internal
964 keywords. They are used to strip out any internal steps from the
965 history tracebacks of the config. Do not modify these keywords to
966 subvert a `~lsst.pex.config.Config` instance's history.
968 Examples
969 --------
970 This is a config with three fields:
972 >>> from lsst.pex.config import Config, Field
973 >>> class DemoConfig(Config):
974 ... fieldA = Field(doc='Field A', dtype=int, default=42)
975 ... fieldB = Field(doc='Field B', dtype=bool, default=True)
976 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
977 ...
978 >>> config = DemoConfig()
980 These are the default values of each field:
982 >>> for name, value in config.iteritems():
983 ... print(f"{name}: {value}")
984 ...
985 fieldA: 42
986 fieldB: True
987 fieldC: 'Hello world'
989 Using this method to update ``fieldA`` and ``fieldC``:
991 >>> config.update(fieldA=13, fieldC='Updated!')
993 Now the values of each field are:
995 >>> for name, value in config.iteritems():
996 ... print(f"{name}: {value}")
997 ...
998 fieldA: 13
999 fieldB: True
1000 fieldC: 'Updated!'
1001 """
1002 at = kw.pop("__at", getCallStack())
1003 label = kw.pop("__label", "update")
1005 for name, value in kw.items():
1006 try:
1007 field = self._fields[name]
1008 field.__set__(self, value, at=at, label=label)
1009 except KeyError:
1010 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
1012 def load(self, filename, root="config"):
1013 """Modify this config in place by executing the Python code in a
1014 configuration file.
1016 Parameters
1017 ----------
1018 filename : `str`
1019 Name of the configuration file. A configuration file is Python
1020 module.
1021 root : `str`, optional
1022 Name of the variable in file that refers to the config being
1023 overridden.
1025 For example, the value of root is ``"config"`` and the file
1026 contains::
1028 config.myField = 5
1030 Then this config's field ``myField`` is set to ``5``.
1032 **Deprecated:** For backwards compatibility, older config files
1033 that use ``root="root"`` instead of ``root="config"`` will be
1034 loaded with a warning printed to `sys.stderr`. This feature will be
1035 removed at some point.
1037 See also
1038 --------
1039 lsst.pex.config.Config.loadFromStream
1040 lsst.pex.config.Config.save
1041 lsst.pex.config.Config.saveFromStream
1042 """
1043 with open(filename, "r") as f:
1044 code = compile(f.read(), filename=filename, mode="exec")
1045 self.loadFromStream(stream=code, root=root, filename=filename)
1047 def loadFromStream(self, stream, root="config", filename=None):
1048 """Modify this Config in place by executing the Python code in the
1049 provided stream.
1051 Parameters
1052 ----------
1053 stream : file-like object, `str`, or compiled string
1054 Stream containing configuration override code.
1055 root : `str`, optional
1056 Name of the variable in file that refers to the config being
1057 overridden.
1059 For example, the value of root is ``"config"`` and the file
1060 contains::
1062 config.myField = 5
1064 Then this config's field ``myField`` is set to ``5``.
1066 **Deprecated:** For backwards compatibility, older config files
1067 that use ``root="root"`` instead of ``root="config"`` will be
1068 loaded with a warning printed to `sys.stderr`. This feature will be
1069 removed at some point.
1070 filename : `str`, optional
1071 Name of the configuration file, or `None` if unknown or contained
1072 in the stream. Used for error reporting.
1074 See also
1075 --------
1076 lsst.pex.config.Config.load
1077 lsst.pex.config.Config.save
1078 lsst.pex.config.Config.saveFromStream
1079 """
1080 with RecordingImporter() as importer:
1081 globals = {"__file__": filename}
1082 try:
1083 local = {root: self}
1084 exec(stream, globals, local)
1085 except NameError as e:
1086 if root == "config" and "root" in e.args[0]:
1087 if filename is None:
1088 # try to determine the file name; a compiled string
1089 # has attribute "co_filename",
1090 # an open file has attribute "name", else give up
1091 filename = getattr(stream, "co_filename", None)
1092 if filename is None:
1093 filename = getattr(stream, "name", "?")
1094 print(f"Config override file {filename!r}"
1095 " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
1096 local = {"root": self}
1097 exec(stream, globals, local)
1098 else:
1099 raise
1101 self._imports.update(importer.getModules())
1103 def save(self, filename, root="config"):
1104 """Save a Python script to the named file, which, when loaded,
1105 reproduces this config.
1107 Parameters
1108 ----------
1109 filename : `str`
1110 Desination filename of this configuration.
1111 root : `str`, optional
1112 Name to use for the root config variable. The same value must be
1113 used when loading (see `lsst.pex.config.Config.load`).
1115 See also
1116 --------
1117 lsst.pex.config.Config.saveToStream
1118 lsst.pex.config.Config.load
1119 lsst.pex.config.Config.loadFromStream
1120 """
1121 d = os.path.dirname(filename)
1122 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1123 self.saveToStream(outfile, root)
1124 # tempfile is hardcoded to create files with mode '0600'
1125 # for an explantion of these antics see:
1126 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1127 umask = os.umask(0o077)
1128 os.umask(umask)
1129 os.chmod(outfile.name, (~umask & 0o666))
1130 # chmod before the move so we get quasi-atomic behavior if the
1131 # source and dest. are on the same filesystem.
1132 # os.rename may not work across filesystems
1133 shutil.move(outfile.name, filename)
1135 def saveToStream(self, outfile, root="config", skipImports=False):
1136 """Save a configuration file to a stream, which, when loaded,
1137 reproduces this config.
1139 Parameters
1140 ----------
1141 outfile : file-like object
1142 Destination file object write the config into. Accepts strings not
1143 bytes.
1144 root
1145 Name to use for the root config variable. The same value must be
1146 used when loading (see `lsst.pex.config.Config.load`).
1147 skipImports : `bool`, optional
1148 If `True` then do not include ``import`` statements in output,
1149 this is to support human-oriented output from ``pipetask`` where
1150 additional clutter is not useful.
1152 See also
1153 --------
1154 lsst.pex.config.Config.save
1155 lsst.pex.config.Config.load
1156 lsst.pex.config.Config.loadFromStream
1157 """
1158 tmp = self._name
1159 self._rename(root)
1160 try:
1161 if not skipImports: 1161 ↛ 1173line 1161 didn't jump to line 1173, because the condition on line 1161 was never false
1162 self._collectImports()
1163 # Remove self from the set, as it is handled explicitly below
1164 self._imports.remove(self.__module__)
1165 configType = type(self)
1166 typeString = _typeStr(configType)
1167 outfile.write(f"import {configType.__module__}\n")
1168 outfile.write(f"assert type({root})=={typeString}, 'config is of type %s.%s instead of "
1169 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n")
1170 for imp in self._imports: 1170 ↛ 1171line 1170 didn't jump to line 1171, because the loop on line 1170 never started
1171 if imp in sys.modules and sys.modules[imp] is not None:
1172 outfile.write(u"import {}\n".format(imp))
1173 self._save(outfile)
1174 finally:
1175 self._rename(tmp)
1177 def freeze(self):
1178 """Make this config, and all subconfigs, read-only.
1179 """
1180 self._frozen = True
1181 for field in self._fields.values():
1182 field.freeze(self)
1184 def _save(self, outfile):
1185 """Save this config to an open stream object.
1187 Parameters
1188 ----------
1189 outfile : file-like object
1190 Destination file object write the config into. Accepts strings not
1191 bytes.
1192 """
1193 for field in self._fields.values():
1194 field.save(outfile, self)
1196 def _collectImports(self):
1197 """Adds module containing self to the list of things to import and
1198 then loops over all the fields in the config calling a corresponding
1199 collect method. The field method will call _collectImports on any
1200 configs it may own and return the set of things to import. This
1201 returned set will be merged with the set of imports for this config
1202 class.
1203 """
1204 self._imports.add(self.__module__)
1205 for name, field in self._fields.items():
1206 field._collectImports(self, self._imports)
1208 def toDict(self):
1209 """Make a dictionary of field names and their values.
1211 Returns
1212 -------
1213 dict_ : `dict`
1214 Dictionary with keys that are `~lsst.pex.config.Field` names.
1215 Values are `~lsst.pex.config.Field` values.
1217 See also
1218 --------
1219 lsst.pex.config.Field.toDict
1221 Notes
1222 -----
1223 This method uses the `~lsst.pex.config.Field.toDict` method of
1224 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1225 implement a ``toDict`` method for *this* method to work.
1226 """
1227 dict_ = {}
1228 for name, field in self._fields.items():
1229 dict_[name] = field.toDict(self)
1230 return dict_
1232 def names(self):
1233 """Get all the field names in the config, recursively.
1235 Returns
1236 -------
1237 names : `list` of `str`
1238 Field names.
1239 """
1240 #
1241 # Rather than sort out the recursion all over again use the
1242 # pre-existing saveToStream()
1243 #
1244 with io.StringIO() as strFd:
1245 self.saveToStream(strFd, "config")
1246 contents = strFd.getvalue()
1247 strFd.close()
1248 #
1249 # Pull the names out of the dumped config
1250 #
1251 keys = []
1252 for line in contents.split("\n"):
1253 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1254 continue
1256 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1257 if mat:
1258 keys.append(mat.group(1))
1260 return keys
1262 def _rename(self, name):
1263 """Rename this config object in its parent `~lsst.pex.config.Config`.
1265 Parameters
1266 ----------
1267 name : `str`
1268 New name for this config in its parent `~lsst.pex.config.Config`.
1270 Notes
1271 -----
1272 This method uses the `~lsst.pex.config.Field.rename` method of
1273 individual `lsst.pex.config.Field` instances.
1274 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1275 method for *this* method to work.
1277 See also
1278 --------
1279 lsst.pex.config.Field.rename
1280 """
1281 self._name = name
1282 for field in self._fields.values():
1283 field.rename(self)
1285 def validate(self):
1286 """Validate the Config, raising an exception if invalid.
1288 Raises
1289 ------
1290 lsst.pex.config.FieldValidationError
1291 Raised if verification fails.
1293 Notes
1294 -----
1295 The base class implementation performs type checks on all fields by
1296 calling their `~lsst.pex.config.Field.validate` methods.
1298 Complex single-field validation can be defined by deriving new Field
1299 types. For convenience, some derived `lsst.pex.config.Field`-types
1300 (`~lsst.pex.config.ConfigField` and
1301 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config`
1302 that handle recursing into subconfigs.
1304 Inter-field relationships should only be checked in derived
1305 `~lsst.pex.config.Config` classes after calling this method, and base
1306 validation is complete.
1307 """
1308 for field in self._fields.values():
1309 field.validate(self)
1311 def formatHistory(self, name, **kwargs):
1312 """Format a configuration field's history to a human-readable format.
1314 Parameters
1315 ----------
1316 name : `str`
1317 Name of a `~lsst.pex.config.Field` in this config.
1318 kwargs
1319 Keyword arguments passed to `lsst.pex.config.history.format`.
1321 Returns
1322 -------
1323 history : `str`
1324 A string containing the formatted history.
1326 See also
1327 --------
1328 lsst.pex.config.history.format
1329 """
1330 import lsst.pex.config.history as pexHist
1331 return pexHist.format(self, name, **kwargs)
1333 history = property(lambda x: x._history) 1333 ↛ exitline 1333 didn't run the lambda on line 1333
1334 """Read-only history.
1335 """
1337 def __setattr__(self, attr, value, at=None, label="assignment"):
1338 """Set an attribute (such as a field's value).
1340 Notes
1341 -----
1342 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1343 locked such that no additional attributes nor properties may be added
1344 to them dynamically.
1346 Although this is not the standard Python behavior, it helps to protect
1347 users from accidentally mispelling a field name, or trying to set a
1348 non-existent field.
1349 """
1350 if attr in self._fields:
1351 if self._fields[attr].deprecated is not None: 1351 ↛ 1352line 1351 didn't jump to line 1352, because the condition on line 1351 was never true
1352 fullname = _joinNamePath(self._name, self._fields[attr].name)
1353 warnings.warn(f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1354 FutureWarning, stacklevel=2)
1355 if at is None: 1355 ↛ 1358line 1355 didn't jump to line 1358, because the condition on line 1355 was never false
1356 at = getCallStack()
1357 # This allows Field descriptors to work.
1358 self._fields[attr].__set__(self, value, at=at, label=label)
1359 elif hasattr(getattr(self.__class__, attr, None), '__set__'): 1359 ↛ 1361line 1359 didn't jump to line 1361, because the condition on line 1359 was never true
1360 # This allows properties and other non-Field descriptors to work.
1361 return object.__setattr__(self, attr, value)
1362 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1362 ↛ 1367line 1362 didn't jump to line 1367, because the condition on line 1362 was never false
1363 # This allows specific private attributes to work.
1364 self.__dict__[attr] = value
1365 else:
1366 # We throw everything else.
1367 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr))
1369 def __delattr__(self, attr, at=None, label="deletion"):
1370 if attr in self._fields:
1371 if at is None:
1372 at = getCallStack()
1373 self._fields[attr].__delete__(self, at=at, label=label)
1374 else:
1375 object.__delattr__(self, attr)
1377 def __eq__(self, other):
1378 if type(other) == type(self): 1378 ↛ 1379line 1378 didn't jump to line 1379, because the condition on line 1378 was never true
1379 for name in self._fields:
1380 thisValue = getattr(self, name)
1381 otherValue = getattr(other, name)
1382 if isinstance(thisValue, float) and math.isnan(thisValue):
1383 if not math.isnan(otherValue):
1384 return False
1385 elif thisValue != otherValue:
1386 return False
1387 return True
1388 return False
1390 def __ne__(self, other):
1391 return not self.__eq__(other)
1393 def __str__(self):
1394 return str(self.toDict())
1396 def __repr__(self):
1397 return "%s(%s)" % (
1398 _typeStr(self),
1399 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None)
1400 )
1402 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
1403 """Compare this configuration to another `~lsst.pex.config.Config` for
1404 equality.
1406 Parameters
1407 ----------
1408 other : `lsst.pex.config.Config`
1409 Other `~lsst.pex.config.Config` object to compare against this
1410 config.
1411 shortcut : `bool`, optional
1412 If `True`, return as soon as an inequality is found. Default is
1413 `True`.
1414 rtol : `float`, optional
1415 Relative tolerance for floating point comparisons.
1416 atol : `float`, optional
1417 Absolute tolerance for floating point comparisons.
1418 output : callable, optional
1419 A callable that takes a string, used (possibly repeatedly) to
1420 report inequalities.
1422 Returns
1423 -------
1424 isEqual : `bool`
1425 `True` when the two `lsst.pex.config.Config` instances are equal.
1426 `False` if there is an inequality.
1428 See also
1429 --------
1430 lsst.pex.config.compareConfigs
1432 Notes
1433 -----
1434 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1435 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1436 are not considered by this method.
1438 Floating point comparisons are performed by `numpy.allclose`.
1439 """
1440 name1 = self._name if self._name is not None else "config"
1441 name2 = other._name if other._name is not None else "config"
1442 name = getComparisonName(name1, name2)
1443 return compareConfigs(name, self, other, shortcut=shortcut,
1444 rtol=rtol, atol=atol, output=output)
1446 @classmethod
1447 def __init_subclass__(cls, **kwargs):
1448 """Run initialization for every subclass.
1450 Specifically registers the subclass with a YAML representer
1451 and YAML constructor (if pyyaml is available)
1452 """
1453 super().__init_subclass__(**kwargs)
1455 if not yaml: 1455 ↛ 1456line 1455 didn't jump to line 1456, because the condition on line 1455 was never true
1456 return
1458 yaml.add_representer(cls, _yaml_config_representer)
1460 @classmethod
1461 def _fromPython(cls, config_py):
1462 """Instantiate a `Config`-subclass from serialized Python form.
1464 Parameters
1465 ----------
1466 config_py : `str`
1467 A serialized form of the Config as created by
1468 `Config.saveToStream`.
1470 Returns
1471 -------
1472 config : `Config`
1473 Reconstructed `Config` instant.
1474 """
1475 cls = _classFromPython(config_py)
1476 return unreduceConfig(cls, config_py)
1479def _classFromPython(config_py):
1480 """Return the Config subclass required by this Config serialization.
1482 Parameters
1483 ----------
1484 config_py : `str`
1485 A serialized form of the Config as created by
1486 `Config.saveToStream`.
1488 Returns
1489 -------
1490 cls : `type`
1491 The `Config` subclass associated with this config.
1492 """
1493 # standard serialization has the form:
1494 # import config.class
1495 # assert type(config)==config.class.Config, ...
1496 # We want to parse these two lines so we can get the class itself
1498 # Do a single regex to avoid large string copies when splitting a
1499 # large config into separate lines.
1500 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py)
1502 if not matches:
1503 first_line, second_line, _ = config_py.split("\n", 2)
1504 raise ValueError("First two lines did not match expected form. Got:\n"
1505 f" - {first_line}\n"
1506 f" - {second_line}")
1508 module_name = matches.group(1)
1509 module = importlib.import_module(module_name)
1511 # Second line
1512 full_name = matches.group(2)
1514 # Remove the module name from the full name
1515 if not full_name.startswith(module_name):
1516 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1518 # if module name is a.b.c and full name is a.b.c.d.E then
1519 # we need to remove a.b.c. and iterate over the remainder
1520 # The +1 is for the extra dot after a.b.c
1521 remainder = full_name[len(module_name)+1:]
1522 components = remainder.split(".")
1523 pytype = module
1524 for component in components:
1525 pytype = getattr(pytype, component)
1526 return pytype
1529def unreduceConfig(cls, stream):
1530 """Create a `~lsst.pex.config.Config` from a stream.
1532 Parameters
1533 ----------
1534 cls : `lsst.pex.config.Config`-type
1535 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1536 with configurations in the ``stream``.
1537 stream : file-like object, `str`, or compiled string
1538 Stream containing configuration override code.
1540 Returns
1541 -------
1542 config : `lsst.pex.config.Config`
1543 Config instance.
1545 See also
1546 --------
1547 lsst.pex.config.Config.loadFromStream
1548 """
1549 config = cls()
1550 config.loadFromStream(stream)
1551 return config