Coverage for python / lsst / pex / config / config.py: 57%
464 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-09 01:14 -0800
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "Config",
31 "ConfigMeta",
32 "Field",
33 "FieldTypeVar",
34 "FieldValidationError",
35 "UnexpectedProxyUsageError",
36)
38import copy
39import importlib
40import io
41import math
42import numbers
43import os
44import re
45import shutil
46import sys
47import tempfile
48import warnings
49from collections.abc import Mapping
50from types import GenericAlias
51from typing import Any, ForwardRef, Generic, TypeVar, cast, overload
53# if YAML is not available that's fine and we simply don't register
54# the yaml representer since we know it won't be used.
55try:
56 import yaml
57except ImportError:
58 yaml = None
60from .callStack import getCallStack, getStackFrame
61from .comparison import compareConfigs, compareScalars, getComparisonName
63if yaml: 63 ↛ 74line 63 didn't jump to line 74 because the condition on line 63 was always true
64 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
66 try:
67 # CLoader is not always available
68 from yaml import CLoader
70 YamlLoaders += (CLoader,)
71 except ImportError:
72 pass
73else:
74 YamlLoaders = ()
75 doImport = None
78class _PexConfigGenericAlias(GenericAlias):
79 """A Subclass of python's GenericAlias used in defining and instantiating
80 Generics.
82 This class differs from `types.GenericAlias` in that it calls a method
83 named _parseTypingArgs defined on Fields. This method gives Field and its
84 subclasses an opportunity to transform type parameters into class key word
85 arguments. Code authors do not need to implement any returns of this object
86 directly, and instead only need implement _parseTypingArgs, if a Field
87 subclass differs from the base class implementation.
89 This class is intended to be an implementation detail, returned from a
90 Field's `__class_getitem__` method.
91 """
93 def __call__(self, *args: Any, **kwds: Any) -> Any:
94 origin_kwargs = self._parseTypingArgs(self.__args__, kwds)
95 return super().__call__(*args, **{**kwds, **origin_kwargs})
98FieldTypeVar = TypeVar("FieldTypeVar")
101class UnexpectedProxyUsageError(TypeError):
102 """Exception raised when a proxy class is used in a context that suggests
103 it should have already been converted to the thing it proxies.
104 """
107def _joinNamePath(prefix=None, name=None, index=None):
108 """Generate nested configuration names."""
109 if not prefix and not name: 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true
110 raise ValueError("Invalid name: cannot be None")
111 elif not name: 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true
112 name = prefix
113 elif prefix and name: 113 ↛ 116line 113 didn't jump to line 116 because the condition on line 113 was always true
114 name = prefix + "." + name
116 if index is not None: 116 ↛ 117line 116 didn't jump to line 117 because the condition on line 116 was never true
117 return f"{name}[{index!r}]"
118 else:
119 return name
122def _autocast(x, dtype):
123 """Cast a value to a type, if appropriate.
125 Parameters
126 ----------
127 x : object
128 A value.
129 dtype : type
130 Data type, such as `float`, `int`, or `str`.
132 Returns
133 -------
134 values : object
135 If appropriate, the returned value is ``x`` cast to the given type
136 ``dtype``. If the cast cannot be performed the original value of
137 ``x`` is returned.
139 Notes
140 -----
141 Will convert numpy scalar types to the standard Python equivalents.
142 """
143 if dtype is float and isinstance(x, numbers.Real): 143 ↛ 145line 143 didn't jump to line 145 because the condition on line 143 was always true
144 return float(x)
145 if dtype is int and isinstance(x, numbers.Integral):
146 return int(x)
147 return x
150def _typeStr(x):
151 """Generate a fully-qualified type name.
153 Returns
154 -------
155 `str`
156 Fully-qualified type name.
158 Notes
159 -----
160 This function is used primarily for writing config files to be executed
161 later upon with the 'load' function.
162 """
163 if hasattr(x, "__module__") and hasattr(x, "__name__"):
164 xtype = x
165 else:
166 xtype = type(x)
167 if xtype.__module__ == "builtins": 167 ↛ 168line 167 didn't jump to line 168 because the condition on line 167 was never true
168 return xtype.__name__
169 else:
170 return f"{xtype.__module__}.{xtype.__name__}"
173if yaml: 173 ↛ 206line 173 didn't jump to line 206 because the condition on line 173 was always true
175 def _yaml_config_representer(dumper, data):
176 """Represent a Config object in a form suitable for YAML.
178 Stores the serialized stream as a scalar block string.
179 """
180 stream = io.StringIO()
181 data.saveToStream(stream)
182 config_py = stream.getvalue()
184 # Strip multiple newlines from the end of the config
185 # This simplifies the YAML to use | and not |+
186 config_py = config_py.rstrip() + "\n"
188 # Trailing spaces force pyyaml to use non-block form.
189 # Remove the trailing spaces so it has no choice
190 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
192 # Store the Python as a simple scalar
193 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
195 def _yaml_config_constructor(loader, node):
196 """Construct a config from YAML."""
197 config_py = loader.construct_scalar(node)
198 return Config._fromPython(config_py)
200 # Register a generic constructor for Config and all subclasses
201 # Need to register for all the loaders we would like to use
202 for loader in YamlLoaders:
203 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
206class ConfigMeta(type):
207 """A metaclass for `lsst.pex.config.Config`.
209 Parameters
210 ----------
211 name : `str`
212 Name to use for class.
213 bases : `~collections.abc.Iterable`
214 Base classes.
215 dict_ : `dict`
216 Additional parameters.
218 Notes
219 -----
220 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
221 class attributes as a class attribute called ``_fields``, and adds
222 the name of each field as an instance variable of the field itself (so you
223 don't have to pass the name of the field to the field constructor).
224 """
226 def __init__(cls, name, bases, dict_):
227 type.__init__(cls, name, bases, dict_)
228 cls._fields = {}
229 cls._source = getStackFrame()
231 def getFields(classtype):
232 fields = {}
233 bases = list(classtype.__bases__)
234 bases.reverse()
235 for b in bases:
236 fields.update(getFields(b))
238 for k, v in classtype.__dict__.items():
239 if isinstance(v, Field):
240 fields[k] = v
241 return fields
243 fields = getFields(cls)
244 for k, v in fields.items():
245 setattr(cls, k, copy.deepcopy(v))
247 def __setattr__(cls, name, value):
248 if isinstance(value, Field):
249 value.name = name
250 cls._fields[name] = value
251 type.__setattr__(cls, name, value)
254class FieldValidationError(ValueError):
255 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
256 particular ``~lsst.pex.config.Config``.
258 Parameters
259 ----------
260 field : `lsst.pex.config.Field`
261 The field that was not valid.
262 config : `lsst.pex.config.Config`
263 The config containing the invalid field.
264 msg : `str`
265 Text describing why the field was not valid.
266 """
268 def __init__(self, field, config, msg):
269 self.fieldType = type(field)
270 """Type of the `~lsst.pex.config.Field` that incurred the error.
271 """
273 self.fieldName = field.name
274 """Name of the `~lsst.pex.config.Field` instance that incurred the
275 error (`str`).
277 See also
278 --------
279 lsst.pex.config.Field.name
280 """
282 self.fullname = _joinNamePath(config._name, field.name)
283 """Fully-qualified name of the `~lsst.pex.config.Field` instance
284 (`str`).
285 """
287 self.history = config.history.setdefault(field.name, [])
288 """Full history of all changes to the `~lsst.pex.config.Field`
289 instance.
290 """
292 self.fieldSource = field.source
293 """File and line number of the `~lsst.pex.config.Field` definition.
294 """
296 self.configSource = config._source
297 error = (
298 f"{self.fieldType.__name__} '{self.fullname}' failed validation: {msg}\n"
299 f"For more information see the Field definition at:\n{self.fieldSource.format()}"
300 f" and the Config definition at:\n{self.configSource.format()}"
301 )
302 super().__init__(error)
305class Field(Generic[FieldTypeVar]):
306 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
307 `complex`, `bool`, and `str` data types.
309 Parameters
310 ----------
311 doc : `str`
312 A description of the field for users.
313 dtype : type, optional
314 The field's data type. ``Field`` only supports basic data types:
315 `int`, `float`, `complex`, `bool`, and `str`. See
316 `Field.supportedTypes`. Optional if supplied as a typing argument to
317 the class.
318 default : object, optional
319 The field's default value.
320 check : callable, optional
321 A callable that is called with the field's value. This callable should
322 return `False` if the value is invalid. More complex inter-field
323 validation can be written as part of the
324 `lsst.pex.config.Config.validate` method.
325 optional : `bool`, optional
326 This sets whether the field is considered optional, and therefore
327 doesn't need to be set by the user. When `False`,
328 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
329 deprecated : None or `str`, optional
330 A description of why this Field is deprecated, including removal date.
331 If not None, the string is appended to the docstring for this Field.
333 Raises
334 ------
335 ValueError
336 Raised when the ``dtype`` parameter is not one of the supported types
337 (see `Field.supportedTypes`).
339 See Also
340 --------
341 ChoiceField
342 ConfigChoiceField
343 ConfigDictField
344 ConfigField
345 ConfigurableField
346 DictField
347 ListField
348 RangeField
349 RegistryField
351 Notes
352 -----
353 ``Field`` instances (including those of any subclass of ``Field``) are used
354 as class attributes of `~lsst.pex.config.Config` subclasses (see the
355 example, below). ``Field`` attributes work like the `property` attributes
356 of classes that implement custom setters and getters. `Field` attributes
357 belong to the class, but operate on the instance. Formally speaking,
358 `Field` attributes are `descriptors
359 <https://docs.python.org/3/howto/descriptor.html>`_.
361 When you access a `Field` attribute on a `Config` instance, you don't
362 get the `Field` instance itself. Instead, you get the value of that field,
363 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
364 container type (like a `lsst.pex.config.List`) depending on the field's
365 type. See the example, below.
367 Fields can be annotated with a type similar to other python classes (python
368 specification `here <https://peps.python.org/pep-0484/#generics>`_ ).
369 See the name field in the Config example below for an example of this.
370 Unlike most other uses in python, this has an effect at type checking *and*
371 runtime. If the type is specified with a class annotation, it will be used
372 as the value of the ``dtype`` in the ``Field`` and there is no need to
373 specify it as an argument during instantiation.
375 There are Some notes on dtype through type annotation syntax. Type
376 annotation syntax supports supplying the argument as a string of a type
377 name. i.e. "float", but this cannot be used to resolve circular references.
378 Type annotation syntax can be used on an identifier in addition to Class
379 assignment i.e. ``variable: Field[str] = Config.someField`` vs
380 ``someField = Field[str](doc="some doc"). However, this syntax is only
381 useful for annotating the type of the identifier (i.e. variable in previous
382 example) and does nothing for assigning the dtype of the ``Field``.
384 Examples
385 --------
386 Instances of ``Field`` should be used as class attributes of
387 `lsst.pex.config.Config` subclasses:
389 >>> from lsst.pex.config import Config, Field
390 >>> class Example(Config):
391 ... myInt = Field("An integer field.", int, default=0)
392 ... name = Field[str](doc="A string Field")
393 >>> print(config.myInt)
394 0
395 >>> config.myInt = 5
396 >>> print(config.myInt)
397 5
398 """
400 name: str
401 """Identifier (variable name) used to refer to a Field within a Config
402 Class.
403 """
405 supportedTypes = {str, bool, float, int, complex}
406 """Supported data types for field values (`set` of types).
407 """
409 @staticmethod
410 def _parseTypingArgs(
411 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
412 ) -> Mapping[str, Any]:
413 """Parse type annotations into keyword constructor arguments.
415 This is a special private method that interprets type arguments (i.e.
416 Field[str]) into keyword arguments to be passed on to the constructor.
418 Subclasses of Field can implement this method to customize how they
419 handle turning type parameters into keyword arguments (see DictField
420 for an example)
422 Parameters
423 ----------
424 params : `tuple` of `type` or `tuple` of str
425 Parameters passed to the type annotation. These will either be
426 types or strings. Strings are to interpreted as forward references
427 and will be treated as such.
428 kwds : `MutableMapping` with keys of `str` and values of `Any`
429 These are the user supplied keywords that are to be passed to the
430 Field constructor.
432 Returns
433 -------
434 kwds : `MutableMapping` with keys of `str` and values of `Any`
435 The mapping of keywords that will be passed onto the constructor
436 of the Field. Should be filled in with any information gleaned
437 from the input parameters.
439 Raises
440 ------
441 ValueError
442 Raised if params is of incorrect length.
443 Raised if a forward reference could not be resolved
444 Raised if there is a conflict between params and values in kwds
445 """
446 if len(params) > 1:
447 raise ValueError("Only single type parameters are supported")
448 unpackedParams = params[0]
449 if isinstance(unpackedParams, str):
450 _typ = ForwardRef(unpackedParams)
451 # type ignore below because typeshed seems to be wrong. It
452 # indicates there are only 2 args, as it was in python 3.8, but
453 # 3.9+ takes 3 args.
454 result = _typ._evaluate(globals(), locals(), recursive_guard=set()) # type: ignore
455 if result is None:
456 raise ValueError("Could not deduce type from input")
457 unpackedParams = cast(type, result)
458 if "dtype" in kwds and kwds["dtype"] != unpackedParams:
459 raise ValueError("Conflicting definition for dtype")
460 elif "dtype" not in kwds:
461 kwds = {**kwds, **{"dtype": unpackedParams}}
462 return kwds
464 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef):
465 return _PexConfigGenericAlias(cls, params)
467 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None):
468 if dtype is None: 468 ↛ 469line 468 didn't jump to line 469 because the condition on line 468 was never true
469 raise ValueError(
470 "dtype must either be supplied as an argument or as a type argument to the class"
471 )
472 if dtype not in self.supportedTypes: 472 ↛ 473line 472 didn't jump to line 473 because the condition on line 472 was never true
473 raise ValueError(f"Unsupported Field dtype {_typeStr(dtype)}")
475 source = getStackFrame()
476 self._setup(
477 doc=doc,
478 dtype=dtype,
479 default=default,
480 check=check,
481 optional=optional,
482 source=source,
483 deprecated=deprecated,
484 )
486 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
487 """Set attributes, usually during initialization."""
488 self.dtype = dtype
489 """Data type for the field.
490 """
492 if not doc: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true
493 raise ValueError("Docstring is empty.")
495 # append the deprecation message to the docstring.
496 if deprecated is not None:
497 doc = f"{doc} Deprecated: {deprecated}"
498 self.doc = doc
499 """A description of the field (`str`).
500 """
502 self.deprecated = deprecated
503 """If not None, a description of why this field is deprecated (`str`).
504 """
506 self.__doc__ = f"{doc} (`{dtype.__name__}`"
507 if optional or default is not None:
508 self.__doc__ += f", default ``{default!r}``"
509 self.__doc__ += ")"
511 self.default = default
512 """Default value for this field.
513 """
515 self.check = check
516 """A user-defined function that validates the value of the field.
517 """
519 self.optional = optional
520 """Flag that determines if the field is required to be set (`bool`).
522 When `False`, `lsst.pex.config.Config.validate` will fail if the
523 field's value is `None`.
524 """
526 self.source = source
527 """The stack frame where this field is defined (`list` of
528 `~lsst.pex.config.callStack.StackFrame`).
529 """
531 def rename(self, instance):
532 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
533 only).
535 Parameters
536 ----------
537 instance : `lsst.pex.config.Config`
538 The config instance that contains this field.
540 Notes
541 -----
542 This method is invoked by the `lsst.pex.config.Config` object that
543 contains this field and should not be called directly.
545 Renaming is only relevant for `~lsst.pex.config.Field` instances that
546 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should
547 rename each subconfig with the full field name as generated by
548 `lsst.pex.config.config._joinNamePath`.
549 """
550 pass
552 def validate(self, instance):
553 """Validate the field (for internal use only).
555 Parameters
556 ----------
557 instance : `lsst.pex.config.Config`
558 The config instance that contains this field.
560 Raises
561 ------
562 lsst.pex.config.FieldValidationError
563 Raised if verification fails.
565 Notes
566 -----
567 This method provides basic validation:
569 - Ensures that the value is not `None` if the field is not optional.
570 - Ensures type correctness.
571 - Ensures that the user-provided ``check`` function is valid.
573 Most `~lsst.pex.config.Field` subclasses should call
574 `lsst.pex.config.Field.validate` if they re-implement
575 `~lsst.pex.config.Field.validate`.
576 """
577 value = self.__get__(instance)
578 if not self.optional and value is None:
579 raise FieldValidationError(self, instance, "Required value cannot be None")
581 def freeze(self, instance):
582 """Make this field read-only (for internal use only).
584 Parameters
585 ----------
586 instance : `lsst.pex.config.Config`
587 The config instance that contains this field.
589 Notes
590 -----
591 Freezing is only relevant for fields that hold subconfigs. Fields which
592 hold subconfigs should freeze each subconfig.
594 **Subclasses should implement this method.**
595 """
596 pass
598 def _validateValue(self, value):
599 """Validate a value.
601 Parameters
602 ----------
603 value : object
604 The value being validated.
606 Raises
607 ------
608 TypeError
609 Raised if the value's type is incompatible with the field's
610 ``dtype``.
611 ValueError
612 Raised if the value is rejected by the ``check`` method.
613 """
614 if value is None: 614 ↛ 615line 614 didn't jump to line 615 because the condition on line 614 was never true
615 return
617 if not isinstance(value, self.dtype): 617 ↛ 618line 617 didn't jump to line 618 because the condition on line 617 was never true
618 msg = (
619 f"Value {value} is of incorrect type {_typeStr(value)}. Expected type {_typeStr(self.dtype)}"
620 )
621 raise TypeError(msg)
622 if self.check is not None and not self.check(value): 622 ↛ 623line 622 didn't jump to line 623 because the condition on line 622 was never true
623 msg = f"Value {value} is not a valid value"
624 raise ValueError(msg)
626 def _collectImports(self, instance, imports):
627 """Call the _collectImports method on all config
628 objects the field may own, and union them with the supplied imports
629 set.
631 Parameters
632 ----------
633 instance : instance or subclass of `lsst.pex.config.Config`
634 A config object that has this field defined on it
635 imports : `set`
636 Set of python modules that need imported after persistence
637 """
638 pass
640 def save(self, outfile, instance):
641 """Save this field to a file (for internal use only).
643 Parameters
644 ----------
645 outfile : file-like object
646 A writeable field handle.
647 instance : `~lsst.pex.config.Config`
648 The `~lsst.pex.config.Config` instance that contains this field.
650 Notes
651 -----
652 This method is invoked by the `~lsst.pex.config.Config` object that
653 contains this field and should not be called directly.
655 The output consists of the documentation string
656 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
657 line is formatted as an assignment: ``{fullname}={value}``.
659 This output can be executed with Python.
660 """
661 value = self.__get__(instance)
662 fullname = _joinNamePath(instance._name, self.name)
664 if self.deprecated and value == self.default: 664 ↛ 665line 664 didn't jump to line 665 because the condition on line 664 was never true
665 return
667 # write full documentation string as comment lines
668 # (i.e. first character is #)
669 doc = "# " + str(self.doc).replace("\n", "\n# ")
670 if isinstance(value, float) and not math.isfinite(value): 670 ↛ 672line 670 didn't jump to line 672 because the condition on line 670 was never true
671 # non-finite numbers need special care
672 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n")
673 else:
674 outfile.write(f"{doc}\n{fullname}={value!r}\n\n")
676 def toDict(self, instance):
677 """Convert the field value so that it can be set as the value of an
678 item in a `dict` (for internal use only).
680 Parameters
681 ----------
682 instance : `~lsst.pex.config.Config`
683 The `~lsst.pex.config.Config` that contains this field.
685 Returns
686 -------
687 value : object
688 The field's value. See *Notes*.
690 Notes
691 -----
692 This method invoked by the owning `~lsst.pex.config.Config` object and
693 should not be called directly.
695 Simple values are passed through. Complex data structures must be
696 manipulated. For example, a `~lsst.pex.config.Field` holding a
697 subconfig should, instead of the subconfig object, return a `dict`
698 where the keys are the field names in the subconfig, and the values are
699 the field values in the subconfig.
700 """
701 return self.__get__(instance)
703 def _copy_storage(self, old: Config, new: Config) -> Any:
704 """Copy the storage for this field in the given field into an object
705 suitable for storage in a new copy of that config.
707 Any frozen storage should be unfrozen.
708 """
709 return copy.deepcopy(old._storage[self.name])
711 @overload
712 def __get__(
713 self, instance: None, owner: Any = None, at: Any = None, label: str = "default"
714 ) -> Field[FieldTypeVar]: ...
716 @overload
717 def __get__(
718 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default"
719 ) -> FieldTypeVar: ...
721 def __get__(self, instance, owner=None, at=None, label="default"):
722 """Define how attribute access should occur on the Config instance
723 This is invoked by the owning config object and should not be called
724 directly.
726 When the field attribute is accessed on a Config class object, it
727 returns the field object itself in order to allow inspection of
728 Config classes.
730 When the field attribute is access on a config instance, the actual
731 value described by the field (and held by the Config instance) is
732 returned.
733 """
734 if instance is None: 734 ↛ 735line 734 didn't jump to line 735 because the condition on line 734 was never true
735 return self
736 else:
737 # try statements are almost free in python if they succeed
738 try:
739 return instance._storage[self.name]
740 except AttributeError:
741 if not isinstance(instance, Config):
742 return self
743 else:
744 raise AttributeError(
745 f"Config {instance} is missing _storage attribute, likely incorrectly initialized"
746 ) from None
748 def __set__(
749 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment"
750 ) -> None:
751 """Set an attribute on the config instance.
753 Parameters
754 ----------
755 instance : `lsst.pex.config.Config`
756 The config instance that contains this field.
757 value : obj
758 Value to set on this field.
759 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\
760 optional
761 The call stack (created by
762 `lsst.pex.config.callStack.getCallStack`).
763 label : `str`, optional
764 Event label for the history.
766 Notes
767 -----
768 This method is invoked by the owning `lsst.pex.config.Config` object
769 and should not be called directly.
771 Derived `~lsst.pex.config.Field` classes may need to override the
772 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
773 should follow the following rules:
775 - Do not allow modification of frozen configs.
776 - Validate the new value **before** modifying the field. Except if the
777 new value is `None`. `None` is special and no attempt should be made
778 to validate it until `lsst.pex.config.Config.validate` is called.
779 - Do not modify the `~lsst.pex.config.Config` instance to contain
780 invalid values.
781 - If the field is modified, update the history of the
782 `lsst.pex.config.field.Field` to reflect the changes.
784 In order to decrease the need to implement this method in derived
785 `~lsst.pex.config.Field` types, value validation is performed in the
786 `lsst.pex.config.Field._validateValue`. If only the validation step
787 differs in the derived `~lsst.pex.config.Field`, it is simpler to
788 implement `lsst.pex.config.Field._validateValue` than to reimplement
789 ``__set__``. More complicated behavior, however, may require
790 reimplementation.
791 """
792 if instance._frozen: 792 ↛ 793line 792 didn't jump to line 793 because the condition on line 792 was never true
793 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
795 history = instance._history.setdefault(self.name, [])
796 if value is not None: 796 ↛ 803line 796 didn't jump to line 803 because the condition on line 796 was always true
797 value = _autocast(value, self.dtype)
798 try:
799 self._validateValue(value)
800 except BaseException as e:
801 raise FieldValidationError(self, instance, str(e)) from e
803 instance._storage[self.name] = value
804 if at is None: 804 ↛ 805line 804 didn't jump to line 805 because the condition on line 804 was never true
805 at = getCallStack()
806 history.append((value, at, label))
808 def __delete__(self, instance, at=None, label="deletion"):
809 """Delete an attribute from a `lsst.pex.config.Config` instance.
811 Parameters
812 ----------
813 instance : `lsst.pex.config.Config`
814 The config instance that contains this field.
815 at : `list` of `lsst.pex.config.callStack.StackFrame`
816 The call stack (created by
817 `lsst.pex.config.callStack.getCallStack`).
818 label : `str`, optional
819 Event label for the history.
821 Notes
822 -----
823 This is invoked by the owning `~lsst.pex.config.Config` object and
824 should not be called directly.
825 """
826 if at is None:
827 at = getCallStack()
828 self.__set__(instance, None, at=at, label=label)
830 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
831 """Compare a field (named `Field.name`) in two
832 `~lsst.pex.config.Config` instances for equality.
834 Parameters
835 ----------
836 instance1 : `lsst.pex.config.Config`
837 Left-hand side `Config` instance to compare.
838 instance2 : `lsst.pex.config.Config`
839 Right-hand side `Config` instance to compare.
840 shortcut : `bool`, optional
841 **Unused.**
842 rtol : `float`, optional
843 Relative tolerance for floating point comparisons.
844 atol : `float`, optional
845 Absolute tolerance for floating point comparisons.
846 output : callable, optional
847 A callable that takes a string, used (possibly repeatedly) to
848 report inequalities.
850 Notes
851 -----
852 This method must be overridden by more complex `Field` subclasses.
854 See Also
855 --------
856 lsst.pex.config.compareScalars
857 """
858 v1 = getattr(instance1, self.name)
859 v2 = getattr(instance2, self.name)
860 name = getComparisonName(
861 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name)
862 )
863 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
866class RecordingImporter:
867 """Importer (for `sys.meta_path`) that records which modules are being
868 imported.
870 *This class does not do any importing itself.*
872 Examples
873 --------
874 Use this class as a context manager to ensure it is properly uninstalled
875 when done:
877 >>> with RecordingImporter() as importer:
878 ... # import stuff
879 ... import numpy as np
880 ... print("Imported: " + importer.getModules())
881 """
883 def __init__(self):
884 self._modules = set()
886 def __enter__(self):
887 self.origMetaPath = sys.meta_path
888 sys.meta_path = [self] + sys.meta_path # type: ignore
889 return self
891 def __exit__(self, *args):
892 self.uninstall()
893 return False # Don't suppress exceptions
895 def uninstall(self):
896 """Uninstall the importer."""
897 sys.meta_path = self.origMetaPath
899 def find_spec(self, fullname, path, target=None):
900 """Find a module.
902 Called as part of the ``import`` chain of events.
904 Parameters
905 ----------
906 fullname : `str`
907 Name of module.
908 path : `list` [`str`]
909 Search path. Unused.
910 target : `~typing.Any`, optional
911 Unused.
912 """
913 self._modules.add(fullname)
914 # Return None because we don't do any importing.
915 return None
917 def getModules(self):
918 """Get the set of modules that were imported.
920 Returns
921 -------
922 modules : `set` of `str`
923 Set of imported module names.
924 """
925 return self._modules
928# type ignore because type checker thinks ConfigMeta is Generic when it is not
929class Config(metaclass=ConfigMeta): # type: ignore
930 """Base class for configuration (*config*) objects.
932 Notes
933 -----
934 A ``Config`` object will usually have several `~lsst.pex.config.Field`
935 instances as class attributes. These are used to define most of the base
936 class behavior.
938 ``Config`` implements a mapping API that provides many `dict`-like methods,
939 such as `keys`, `values`, and `items`. ``Config`` instances also support
940 the ``in`` operator to test if a field is in the config. Unlike a `dict`,
941 ``Config`` classes are not subscriptable. Instead, access individual
942 fields as attributes of the configuration instance.
944 Examples
945 --------
946 Config classes are subclasses of ``Config`` that have
947 `~lsst.pex.config.Field` instances (or instances of
948 `~lsst.pex.config.Field` subclasses) as class attributes:
950 >>> from lsst.pex.config import Config, Field, ListField
951 >>> class DemoConfig(Config):
952 ... intField = Field(doc="An integer field", dtype=int, default=42)
953 ... listField = ListField(
954 ... doc="List of favorite beverages.",
955 ... dtype=str,
956 ... default=["coffee", "green tea", "water"],
957 ... )
958 >>> config = DemoConfig()
960 Configs support many `dict`-like APIs:
962 >>> config.keys()
963 ['intField', 'listField']
964 >>> "intField" in config
965 True
967 Individual fields can be accessed as attributes of the configuration:
969 >>> config.intField
970 42
971 >>> config.listField.append("earl grey tea")
972 >>> print(config.listField)
973 ['coffee', 'green tea', 'water', 'earl grey tea']
974 """
976 _storage: dict[str, Any]
977 _fields: dict[str, Field]
978 _history: dict[str, list[Any]]
979 _imports: set[Any]
981 def __iter__(self):
982 """Iterate over fields."""
983 return self._fields.__iter__()
985 def keys(self):
986 """Get field names.
988 Returns
989 -------
990 names : `~collections.abc.KeysView`
991 List of `lsst.pex.config.Field` names.
992 """
993 return self._storage.keys()
995 def values(self):
996 """Get field values.
998 Returns
999 -------
1000 values : `~collections.abc.ValuesView`
1001 Iterator of field values.
1002 """
1003 return self._storage.values()
1005 def items(self):
1006 """Get configurations as ``(field name, field value)`` pairs.
1008 Returns
1009 -------
1010 items : `~collections.abc.ItemsView`
1011 Iterator of tuples for each configuration. Tuple items are:
1013 0. Field name.
1014 1. Field value.
1015 """
1016 return self._storage.items()
1018 def __contains__(self, name):
1019 """Return `True` if the specified field exists in this config.
1021 Parameters
1022 ----------
1023 name : `str`
1024 Field name to test for.
1026 Returns
1027 -------
1028 in : `bool`
1029 `True` if the specified field exists in the config.
1030 """
1031 return self._storage.__contains__(name)
1033 def __new__(cls, *args, **kw):
1034 """Allocate a new `lsst.pex.config.Config` object.
1036 In order to ensure that all Config object are always in a proper state
1037 when handed to users or to derived `~lsst.pex.config.Config` classes,
1038 some attributes are handled at allocation time rather than at
1039 initialization.
1041 This ensures that even if a derived `~lsst.pex.config.Config` class
1042 implements ``__init__``, its author does not need to be concerned about
1043 when or even the base ``Config.__init__`` should be called.
1044 """
1045 name = kw.pop("__name", None)
1046 at = kw.pop("__at", getCallStack())
1047 # remove __label and ignore it
1048 kw.pop("__label", "default")
1050 instance = object.__new__(cls)
1051 instance._frozen = False
1052 instance._name = name
1053 instance._storage = {}
1054 instance._history = {}
1055 instance._imports = set()
1056 # load up defaults
1057 for field in instance._fields.values():
1058 instance._history[field.name] = []
1059 field.__set__(instance, field.default, at=at + [field.source], label="default")
1060 # set custom default-overrides
1061 instance.setDefaults()
1062 # set constructor overrides
1063 instance.update(__at=at, **kw)
1064 return instance
1066 def copy(self) -> Config:
1067 """Return a deep copy of this config.
1069 Notes
1070 -----
1071 The returned config object is not frozen, even if the original was.
1072 If a nested config object is copied, it retains the name from its
1073 original hierarchy.
1075 Nested objects are only shared between the new and old configs if they
1076 are not possible to modify via the config's interfaces (e.g. entries
1077 in the the history list are not copied, but the lists themselves are,
1078 so modifications to one copy do not modify the other).
1079 """
1080 instance = object.__new__(type(self))
1081 instance._frozen = False
1082 instance._name = self._name
1083 instance._history = {k: list(v) for k, v in self._history.items()}
1084 instance._imports = set(self._imports)
1085 # Important to set up storage last, since fields sometimes store
1086 # proxy objects that reference their parent (especially for history).
1087 instance._storage = {k: self._fields[k]._copy_storage(self, instance) for k in self._storage}
1088 return instance
1090 def __reduce__(self):
1091 """Reduction for pickling (function with arguments to reproduce).
1093 We need to condense and reconstitute the `~lsst.pex.config.Config`,
1094 since it may contain lambdas (as the ``check`` elements) that cannot
1095 be pickled.
1096 """
1097 # The stream must be in characters to match the API but pickle
1098 # requires bytes
1099 stream = io.StringIO()
1100 self.saveToStream(stream)
1101 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
1103 def setDefaults(self):
1104 """Subclass hook for computing defaults.
1106 Notes
1107 -----
1108 Derived `~lsst.pex.config.Config` classes that must compute defaults
1109 rather than using the `~lsst.pex.config.Field` instances's defaults
1110 should do so here. To correctly use inherited defaults,
1111 implementations of ``setDefaults`` must call their base class's
1112 ``setDefaults``.
1113 """
1114 pass
1116 def update(self, **kw):
1117 """Update values of fields specified by the keyword arguments.
1119 Parameters
1120 ----------
1121 **kw
1122 Keywords are configuration field names. Values are configuration
1123 field values.
1125 Notes
1126 -----
1127 The ``__at`` and ``__label`` keyword arguments are special internal
1128 keywords. They are used to strip out any internal steps from the
1129 history tracebacks of the config. Do not modify these keywords to
1130 subvert a `~lsst.pex.config.Config` instance's history.
1132 Examples
1133 --------
1134 This is a config with three fields:
1136 >>> from lsst.pex.config import Config, Field
1137 >>> class DemoConfig(Config):
1138 ... fieldA = Field(doc="Field A", dtype=int, default=42)
1139 ... fieldB = Field(doc="Field B", dtype=bool, default=True)
1140 ... fieldC = Field(doc="Field C", dtype=str, default="Hello world")
1141 >>> config = DemoConfig()
1143 These are the default values of each field:
1145 >>> for name, value in config.iteritems():
1146 ... print(f"{name}: {value}")
1147 fieldA: 42
1148 fieldB: True
1149 fieldC: 'Hello world'
1151 Using this method to update ``fieldA`` and ``fieldC``:
1153 >>> config.update(fieldA=13, fieldC="Updated!")
1155 Now the values of each field are:
1157 >>> for name, value in config.iteritems():
1158 ... print(f"{name}: {value}")
1159 fieldA: 13
1160 fieldB: True
1161 fieldC: 'Updated!'
1162 """
1163 at = kw.pop("__at", getCallStack())
1164 label = kw.pop("__label", "update")
1166 for name, value in kw.items():
1167 try:
1168 field = self._fields[name]
1169 field.__set__(self, value, at=at, label=label)
1170 except KeyError as e:
1171 e.add_note(f"No field of name {name} exists in config type {_typeStr(self)}")
1172 raise
1174 def load(self, filename, root="config"):
1175 """Modify this config in place by executing the Python code in a
1176 configuration file.
1178 Parameters
1179 ----------
1180 filename : `str`
1181 Name of the configuration file. A configuration file is Python
1182 module.
1183 root : `str`, optional
1184 Name of the variable in file that refers to the config being
1185 overridden.
1187 For example, the value of root is ``"config"`` and the file
1188 contains::
1190 config.myField = 5
1192 Then this config's field ``myField`` is set to ``5``.
1194 See Also
1195 --------
1196 lsst.pex.config.Config.loadFromStream
1197 lsst.pex.config.Config.loadFromString
1198 lsst.pex.config.Config.save
1199 lsst.pex.config.Config.saveToStream
1200 lsst.pex.config.Config.saveToString
1201 """
1202 with open(filename) as f:
1203 code = compile(f.read(), filename=filename, mode="exec")
1204 self.loadFromString(code, root=root, filename=filename)
1206 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None):
1207 """Modify this Config in place by executing the Python code in the
1208 provided stream.
1210 Parameters
1211 ----------
1212 stream : file-like object, `str`, `bytes`, or `~types.CodeType`
1213 Stream containing configuration override code. If this is a
1214 code object, it should be compiled with ``mode="exec"``.
1215 root : `str`, optional
1216 Name of the variable in file that refers to the config being
1217 overridden.
1219 For example, the value of root is ``"config"`` and the file
1220 contains::
1222 config.myField = 5
1224 Then this config's field ``myField`` is set to ``5``.
1225 filename : `str`, optional
1226 Name of the configuration file, or `None` if unknown or contained
1227 in the stream. Used for error reporting.
1228 extraLocals : `dict` of `str` to `object`, optional
1229 Any extra variables to include in local scope when loading.
1231 Notes
1232 -----
1233 For backwards compatibility reasons, this method accepts strings, bytes
1234 and code objects as well as file-like objects. New code should use
1235 `loadFromString` instead for most of these types.
1237 See Also
1238 --------
1239 lsst.pex.config.Config.load
1240 lsst.pex.config.Config.loadFromString
1241 lsst.pex.config.Config.save
1242 lsst.pex.config.Config.saveToStream
1243 lsst.pex.config.Config.saveToString
1244 """
1245 if hasattr(stream, "read"): 1245 ↛ 1246line 1245 didn't jump to line 1246 because the condition on line 1245 was never true
1246 if filename is None:
1247 filename = getattr(stream, "name", "?")
1248 code = compile(stream.read(), filename=filename, mode="exec")
1249 else:
1250 code = stream
1251 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1253 def loadFromString(self, code, root="config", filename=None, extraLocals=None):
1254 """Modify this Config in place by executing the Python code in the
1255 provided string.
1257 Parameters
1258 ----------
1259 code : `str`, `bytes`, or `~types.CodeType`
1260 Stream containing configuration override code.
1261 root : `str`, optional
1262 Name of the variable in file that refers to the config being
1263 overridden.
1265 For example, the value of root is ``"config"`` and the file
1266 contains::
1268 config.myField = 5
1270 Then this config's field ``myField`` is set to ``5``.
1271 filename : `str`, optional
1272 Name of the configuration file, or `None` if unknown or contained
1273 in the stream. Used for error reporting.
1274 extraLocals : `dict` of `str` to `object`, optional
1275 Any extra variables to include in local scope when loading.
1277 Raises
1278 ------
1279 ValueError
1280 Raised if a key in extraLocals is the same value as the value of
1281 the root argument.
1283 See Also
1284 --------
1285 lsst.pex.config.Config.load
1286 lsst.pex.config.Config.loadFromStream
1287 lsst.pex.config.Config.save
1288 lsst.pex.config.Config.saveToStream
1289 lsst.pex.config.Config.saveToString
1290 """
1291 if filename is None: 1291 ↛ 1295line 1291 didn't jump to line 1295 because the condition on line 1291 was always true
1292 # try to determine the file name; a compiled string
1293 # has attribute "co_filename",
1294 filename = getattr(code, "co_filename", "?")
1295 with RecordingImporter() as importer:
1296 globals = {"__file__": filename}
1297 local = {root: self}
1298 if extraLocals is not None: 1298 ↛ 1300line 1298 didn't jump to line 1300 because the condition on line 1298 was never true
1299 # verify the value of root was not passed as extra local args
1300 if root in extraLocals:
1301 raise ValueError(
1302 f"{root} is reserved and cannot be used as a variable name in extraLocals"
1303 )
1304 local.update(extraLocals)
1305 exec(code, globals, local)
1307 self._imports.update(importer.getModules())
1309 def save(self, filename, root="config"):
1310 """Save a Python script to the named file, which, when loaded,
1311 reproduces this config.
1313 Parameters
1314 ----------
1315 filename : `str`
1316 Desination filename of this configuration.
1317 root : `str`, optional
1318 Name to use for the root config variable. The same value must be
1319 used when loading (see `lsst.pex.config.Config.load`).
1321 See Also
1322 --------
1323 lsst.pex.config.Config.saveToStream
1324 lsst.pex.config.Config.saveToString
1325 lsst.pex.config.Config.load
1326 lsst.pex.config.Config.loadFromStream
1327 lsst.pex.config.Config.loadFromString
1328 """
1329 d = os.path.dirname(filename)
1330 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1331 self.saveToStream(outfile, root)
1332 # tempfile is hardcoded to create files with mode '0600'
1333 # for an explantion of these antics see:
1334 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1335 umask = os.umask(0o077)
1336 os.umask(umask)
1337 os.chmod(outfile.name, (~umask & 0o666))
1338 # chmod before the move so we get quasi-atomic behavior if the
1339 # source and dest. are on the same filesystem.
1340 # os.rename may not work across filesystems
1341 shutil.move(outfile.name, filename)
1343 def saveToString(self, skipImports=False):
1344 """Return the Python script form of this configuration as an executable
1345 string.
1347 Parameters
1348 ----------
1349 skipImports : `bool`, optional
1350 If `True` then do not include ``import`` statements in output,
1351 this is to support human-oriented output from ``pipetask`` where
1352 additional clutter is not useful.
1354 Returns
1355 -------
1356 code : `str`
1357 A code string readable by `loadFromString`.
1359 See Also
1360 --------
1361 lsst.pex.config.Config.save
1362 lsst.pex.config.Config.saveToStream
1363 lsst.pex.config.Config.load
1364 lsst.pex.config.Config.loadFromStream
1365 lsst.pex.config.Config.loadFromString
1366 """
1367 buffer = io.StringIO()
1368 self.saveToStream(buffer, skipImports=skipImports)
1369 return buffer.getvalue()
1371 def saveToStream(self, outfile, root="config", skipImports=False):
1372 """Save a configuration file to a stream, which, when loaded,
1373 reproduces this config.
1375 Parameters
1376 ----------
1377 outfile : file-like object
1378 Destination file object write the config into. Accepts strings not
1379 bytes.
1380 root : `str`, optional
1381 Name to use for the root config variable. The same value must be
1382 used when loading (see `lsst.pex.config.Config.load`).
1383 skipImports : `bool`, optional
1384 If `True` then do not include ``import`` statements in output,
1385 this is to support human-oriented output from ``pipetask`` where
1386 additional clutter is not useful.
1388 See Also
1389 --------
1390 lsst.pex.config.Config.save
1391 lsst.pex.config.Config.saveToString
1392 lsst.pex.config.Config.load
1393 lsst.pex.config.Config.loadFromStream
1394 lsst.pex.config.Config.loadFromString
1395 """
1396 tmp = self._name
1397 self._rename(root)
1398 try:
1399 if not skipImports: 1399 ↛ 1416line 1399 didn't jump to line 1416 because the condition on line 1399 was always true
1400 self._collectImports()
1401 # Remove self from the set, as it is handled explicitly below
1402 self._imports.remove(self.__module__)
1403 configType = type(self)
1404 typeString = _typeStr(configType)
1405 outfile.write(f"import {configType.__module__}\n")
1406 # We are required to write this on a single line because
1407 # of later regex matching, rather than adopting black style
1408 # formatting.
1409 outfile.write(
1410 f'assert type({root}) is {typeString}, f"config is of type '
1411 f'{{type({root}).__module__}}.{{type({root}).__name__}} instead of {typeString}"\n\n'
1412 )
1413 for imp in sorted(self._imports): 1413 ↛ 1414line 1413 didn't jump to line 1414 because the loop on line 1413 never started
1414 if imp in sys.modules and sys.modules[imp] is not None:
1415 outfile.write(f"import {imp}\n")
1416 self._save(outfile)
1417 finally:
1418 self._rename(tmp)
1420 def freeze(self):
1421 """Make this config, and all subconfigs, read-only."""
1422 self._frozen = True
1423 for field in self._fields.values():
1424 field.freeze(self)
1426 def _save(self, outfile):
1427 """Save this config to an open stream object.
1429 Parameters
1430 ----------
1431 outfile : file-like object
1432 Destination file object write the config into. Accepts strings not
1433 bytes.
1434 """
1435 for field in self._fields.values():
1436 field.save(outfile, self)
1438 def _collectImports(self):
1439 """Add module containing self to the list of things to import and
1440 then loops over all the fields in the config calling a corresponding
1441 collect method.
1443 The field method will call _collectImports on any
1444 configs it may own and return the set of things to import. This
1445 returned set will be merged with the set of imports for this config
1446 class.
1447 """
1448 self._imports.add(self.__module__)
1449 for field in self._fields.values():
1450 field._collectImports(self, self._imports)
1452 def toDict(self):
1453 """Make a dictionary of field names and their values.
1455 Returns
1456 -------
1457 dict_ : `dict`
1458 Dictionary with keys that are `~lsst.pex.config.Field` names.
1459 Values are `~lsst.pex.config.Field` values.
1461 See Also
1462 --------
1463 lsst.pex.config.Field.toDict
1465 Notes
1466 -----
1467 This method uses the `~lsst.pex.config.Field.toDict` method of
1468 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1469 implement a ``toDict`` method for *this* method to work.
1470 """
1471 dict_ = {}
1472 for name, field in self._fields.items():
1473 dict_[name] = field.toDict(self)
1474 return dict_
1476 def names(self):
1477 """Get all the field names in the config, recursively.
1479 Returns
1480 -------
1481 names : `list` of `str`
1482 Field names.
1483 """
1484 #
1485 # Rather than sort out the recursion all over again use the
1486 # pre-existing saveToStream()
1487 #
1488 with io.StringIO() as strFd:
1489 self.saveToStream(strFd, "config")
1490 contents = strFd.getvalue()
1491 strFd.close()
1492 #
1493 # Pull the names out of the dumped config
1494 #
1495 keys = []
1496 for line in contents.split("\n"):
1497 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1498 continue
1500 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1501 if mat:
1502 keys.append(mat.group(1))
1504 return keys
1506 def _rename(self, name):
1507 """Rename this config object in its parent `~lsst.pex.config.Config`.
1509 Parameters
1510 ----------
1511 name : `str`
1512 New name for this config in its parent `~lsst.pex.config.Config`.
1514 Notes
1515 -----
1516 This method uses the `~lsst.pex.config.Field.rename` method of
1517 individual `lsst.pex.config.Field` instances.
1518 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1519 method for *this* method to work.
1521 See Also
1522 --------
1523 lsst.pex.config.Field.rename
1524 """
1525 self._name = name
1526 for field in self._fields.values():
1527 field.rename(self)
1529 def validate(self):
1530 """Validate the Config, raising an exception if invalid.
1532 Raises
1533 ------
1534 lsst.pex.config.FieldValidationError
1535 Raised if verification fails.
1537 Notes
1538 -----
1539 The base class implementation performs type checks on all fields by
1540 calling their `~lsst.pex.config.Field.validate` methods.
1542 Complex single-field validation can be defined by deriving new Field
1543 types. For convenience, some derived `lsst.pex.config.Field`-types
1544 (`~lsst.pex.config.ConfigField` and
1545 `~lsst.pex.config.ConfigChoiceField`) are defined in
1546 ``lsst.pex.config`` that handle recursing into subconfigs.
1548 Inter-field relationships should only be checked in derived
1549 `~lsst.pex.config.Config` classes after calling this method, and base
1550 validation is complete.
1551 """
1552 for field in self._fields.values():
1553 field.validate(self)
1555 def formatHistory(self, name, **kwargs):
1556 """Format a configuration field's history to a human-readable format.
1558 Parameters
1559 ----------
1560 name : `str`
1561 Name of a `~lsst.pex.config.Field` in this config.
1562 **kwargs
1563 Keyword arguments passed to `lsst.pex.config.history.format`.
1565 Returns
1566 -------
1567 history : `str`
1568 A string containing the formatted history.
1570 See Also
1571 --------
1572 lsst.pex.config.history.format
1573 """
1574 import lsst.pex.config.history as pexHist
1576 return pexHist.format(self, name, **kwargs)
1578 history = property(lambda x: x._history)
1579 """Read-only history.
1580 """
1582 def __setattr__(self, attr, value, at=None, label="assignment"):
1583 """Set an attribute (such as a field's value).
1585 Notes
1586 -----
1587 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1588 locked such that no additional attributes nor properties may be added
1589 to them dynamically.
1591 Although this is not the standard Python behavior, it helps to protect
1592 users from accidentally mispelling a field name, or trying to set a
1593 non-existent field.
1594 """
1595 if attr in self._fields:
1596 if self._fields[attr].deprecated is not None: 1596 ↛ 1597line 1596 didn't jump to line 1597 because the condition on line 1596 was never true
1597 fullname = _joinNamePath(self._name, self._fields[attr].name)
1598 warnings.warn(
1599 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1600 FutureWarning,
1601 stacklevel=2,
1602 )
1603 if at is None: 1603 ↛ 1606line 1603 didn't jump to line 1606 because the condition on line 1603 was always true
1604 at = getCallStack()
1605 # This allows Field descriptors to work.
1606 self._fields[attr].__set__(self, value, at=at, label=label)
1607 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1607 ↛ 1609line 1607 didn't jump to line 1609 because the condition on line 1607 was never true
1608 # This allows properties and other non-Field descriptors to work.
1609 return object.__setattr__(self, attr, value)
1610 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1610 ↛ 1615line 1610 didn't jump to line 1615 because the condition on line 1610 was always true
1611 # This allows specific private attributes to work.
1612 self.__dict__[attr] = value
1613 else:
1614 # We throw everything else.
1615 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
1617 def __delattr__(self, attr, at=None, label="deletion"):
1618 if attr in self._fields:
1619 if at is None:
1620 at = getCallStack()
1621 self._fields[attr].__delete__(self, at=at, label=label)
1622 else:
1623 object.__delattr__(self, attr)
1625 def __eq__(self, other):
1626 if type(other) is type(self): 1626 ↛ 1627line 1626 didn't jump to line 1627 because the condition on line 1626 was never true
1627 for name in self._fields:
1628 thisValue = getattr(self, name)
1629 otherValue = getattr(other, name)
1630 if isinstance(thisValue, float) and math.isnan(thisValue):
1631 if not math.isnan(otherValue):
1632 return False
1633 elif thisValue != otherValue:
1634 return False
1635 return True
1636 return False
1638 def __ne__(self, other):
1639 return not self.__eq__(other)
1641 def __str__(self):
1642 return str(self.toDict())
1644 def __repr__(self):
1645 return "{}({})".format(
1646 _typeStr(self),
1647 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None),
1648 )
1650 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None):
1651 """Compare this configuration to another `~lsst.pex.config.Config` for
1652 equality.
1654 Parameters
1655 ----------
1656 other : `lsst.pex.config.Config`
1657 Other `~lsst.pex.config.Config` object to compare against this
1658 config.
1659 shortcut : `bool`, optional
1660 If `True`, return as soon as an inequality is found. Default is
1661 `True`.
1662 rtol : `float`, optional
1663 Relative tolerance for floating point comparisons.
1664 atol : `float`, optional
1665 Absolute tolerance for floating point comparisons.
1666 output : callable, optional
1667 A callable that takes a string, used (possibly repeatedly) to
1668 report inequalities.
1670 Returns
1671 -------
1672 isEqual : `bool`
1673 `True` when the two `lsst.pex.config.Config` instances are equal.
1674 `False` if there is an inequality.
1676 See Also
1677 --------
1678 lsst.pex.config.compareConfigs
1680 Notes
1681 -----
1682 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1683 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1684 are not considered by this method.
1686 Floating point comparisons are performed by `numpy.allclose`.
1687 """
1688 name1 = self._name if self._name is not None else "config"
1689 name2 = other._name if other._name is not None else "config"
1690 name = getComparisonName(name1, name2)
1691 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output)
1693 @classmethod
1694 def __init_subclass__(cls, **kwargs):
1695 """Run initialization for every subclass.
1697 Specifically registers the subclass with a YAML representer
1698 and YAML constructor (if pyyaml is available)
1699 """
1700 super().__init_subclass__(**kwargs)
1702 if not yaml: 1702 ↛ 1703line 1702 didn't jump to line 1703 because the condition on line 1702 was never true
1703 return
1705 yaml.add_representer(cls, _yaml_config_representer)
1707 @classmethod
1708 def _fromPython(cls, config_py):
1709 """Instantiate a `Config`-subclass from serialized Python form.
1711 Parameters
1712 ----------
1713 config_py : `str`
1714 A serialized form of the Config as created by
1715 `Config.saveToStream`.
1717 Returns
1718 -------
1719 config : `Config`
1720 Reconstructed `Config` instant.
1721 """
1722 cls = _classFromPython(config_py)
1723 return unreduceConfig(cls, config_py)
1726def _classFromPython(config_py):
1727 """Return the Config subclass required by this Config serialization.
1729 Parameters
1730 ----------
1731 config_py : `str`
1732 A serialized form of the Config as created by
1733 `Config.saveToStream`.
1735 Returns
1736 -------
1737 cls : `type`
1738 The `Config` subclass associated with this config.
1739 """
1740 # standard serialization has the form:
1741 # import config.class
1742 # assert type(config) is config.class.Config, ...
1743 # Older files use "type(config)==" instead.
1744 # We want to parse these two lines so we can get the class itself
1746 # Do a single regex to avoid large string copies when splitting a
1747 # large config into separate lines.
1748 # The assert regex cannot be greedy because the assert error string
1749 # can include both "," and " is ".
1750 matches = re.search(r"^import ([\w.]+)\nassert type\(\S+\)(?:\s*==\s*| is )(.*?),", config_py)
1752 if not matches:
1753 first_line, second_line, _ = config_py.split("\n", 2)
1754 raise ValueError(
1755 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}"
1756 )
1758 module_name = matches.group(1)
1759 module = importlib.import_module(module_name)
1761 # Second line
1762 full_name = matches.group(2)
1764 # Remove the module name from the full name
1765 if not full_name.startswith(module_name):
1766 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1768 # if module name is a.b.c and full name is a.b.c.d.E then
1769 # we need to remove a.b.c. and iterate over the remainder
1770 # The +1 is for the extra dot after a.b.c
1771 remainder = full_name[len(module_name) + 1 :]
1772 components = remainder.split(".")
1773 pytype = module
1774 for component in components:
1775 pytype = getattr(pytype, component)
1776 return pytype
1779def unreduceConfig(cls_, stream):
1780 """Create a `~lsst.pex.config.Config` from a stream.
1782 Parameters
1783 ----------
1784 cls_ : `lsst.pex.config.Config`-type
1785 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1786 with configurations in the ``stream``.
1787 stream : file-like object, `str`, or `~types.CodeType`
1788 Stream containing configuration override code.
1790 Returns
1791 -------
1792 config : `lsst.pex.config.Config`
1793 Config instance.
1795 See Also
1796 --------
1797 lsst.pex.config.Config.loadFromStream
1798 """
1799 config = cls_()
1800 config.loadFromStream(stream)
1801 return config