Coverage for python / lsst / pex / config / config.py: 56%
500 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-17 08:53 +0000
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "Config",
31 "ConfigMeta",
32 "Field",
33 "FieldTypeVar",
34 "FieldValidationError",
35 "UnexpectedProxyUsageError",
36)
38import copy
39import importlib
40import io
41import logging
42import math
43import numbers
44import os
45import re
46import shutil
47import sys
48import tempfile
49import warnings
50from collections.abc import Mapping
51from contextlib import contextmanager
52from contextvars import ContextVar
53from types import GenericAlias
54from typing import Any, ForwardRef, Generic, TypeVar, cast, overload
56from lsst.resources import ResourcePath, ResourcePathExpression
58# if YAML is not available that's fine and we simply don't register
59# the yaml representer since we know it won't be used.
60try:
61 import yaml
62except ImportError:
63 yaml = None
65from .callStack import getCallStack, getStackFrame
66from .comparison import compareConfigs, compareScalars, getComparisonName
68if yaml: 68 ↛ 79line 68 didn't jump to line 79 because the condition on line 68 was always true
69 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
71 try:
72 # CLoader is not always available
73 from yaml import CLoader
75 YamlLoaders += (CLoader,)
76 except ImportError:
77 pass
78else:
79 YamlLoaders = ()
80 doImport = None
82_LOG = logging.getLogger(__name__)
85# Tracks the current config directory for the current context.
86_config_dir_stack: ContextVar[ResourcePath | None] = ContextVar("_config_dir_stack", default=None)
89def _get_config_root() -> ResourcePath | None:
90 return _config_dir_stack.get()
93@contextmanager
94def _push_config_root(dirname: ResourcePath):
95 token = _config_dir_stack.set(dirname)
96 try:
97 yield
98 finally:
99 _config_dir_stack.reset(token)
102class _PexConfigGenericAlias(GenericAlias):
103 """A Subclass of python's GenericAlias used in defining and instantiating
104 Generics.
106 This class differs from `types.GenericAlias` in that it calls a method
107 named _parseTypingArgs defined on Fields. This method gives Field and its
108 subclasses an opportunity to transform type parameters into class key word
109 arguments. Code authors do not need to implement any returns of this object
110 directly, and instead only need implement _parseTypingArgs, if a Field
111 subclass differs from the base class implementation.
113 This class is intended to be an implementation detail, returned from a
114 Field's `__class_getitem__` method.
115 """
117 def __call__(self, *args: Any, **kwds: Any) -> Any:
118 origin_kwargs = self._parseTypingArgs(self.__args__, kwds)
119 return super().__call__(*args, **{**kwds, **origin_kwargs})
122FieldTypeVar = TypeVar("FieldTypeVar")
125class UnexpectedProxyUsageError(TypeError):
126 """Exception raised when a proxy class is used in a context that suggests
127 it should have already been converted to the thing it proxies.
128 """
131def _joinNamePath(prefix=None, name=None, index=None):
132 """Generate nested configuration names."""
133 if not prefix and not name: 133 ↛ 134line 133 didn't jump to line 134 because the condition on line 133 was never true
134 raise ValueError("Invalid name: cannot be None")
135 elif not name: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 name = prefix
137 elif prefix and name: 137 ↛ 140line 137 didn't jump to line 140 because the condition on line 137 was always true
138 name = prefix + "." + name
140 if index is not None: 140 ↛ 141line 140 didn't jump to line 141 because the condition on line 140 was never true
141 return f"{name}[{index!r}]"
142 else:
143 return name
146def _autocast(x, dtype):
147 """Cast a value to a type, if appropriate.
149 Parameters
150 ----------
151 x : `object`
152 A value.
153 dtype : type
154 Data type, such as `float`, `int`, or `str`.
156 Returns
157 -------
158 values : `object`
159 If appropriate, the returned value is ``x`` cast to the given type
160 ``dtype``. If the cast cannot be performed the original value of
161 ``x`` is returned.
163 Notes
164 -----
165 Will convert numpy scalar types to the standard Python equivalents.
166 """
167 if dtype is float and isinstance(x, numbers.Real): 167 ↛ 169line 167 didn't jump to line 169 because the condition on line 167 was always true
168 return float(x)
169 if dtype is int and isinstance(x, numbers.Integral):
170 return int(x)
171 return x
174def _typeStr(x):
175 """Generate a fully-qualified type name.
177 Returns
178 -------
179 `str`
180 Fully-qualified type name.
182 Notes
183 -----
184 This function is used primarily for writing config files to be executed
185 later upon with the 'load' function.
186 """
187 if hasattr(x, "__module__") and hasattr(x, "__name__"):
188 xtype = x
189 else:
190 xtype = type(x)
191 if xtype.__module__ == "builtins": 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true
192 return xtype.__name__
193 else:
194 return f"{xtype.__module__}.{xtype.__name__}"
197if yaml: 197 ↛ 230line 197 didn't jump to line 230 because the condition on line 197 was always true
199 def _yaml_config_representer(dumper, data):
200 """Represent a Config object in a form suitable for YAML.
202 Stores the serialized stream as a scalar block string.
203 """
204 stream = io.StringIO()
205 data.saveToStream(stream)
206 config_py = stream.getvalue()
208 # Strip multiple newlines from the end of the config
209 # This simplifies the YAML to use | and not |+
210 config_py = config_py.rstrip() + "\n"
212 # Trailing spaces force pyyaml to use non-block form.
213 # Remove the trailing spaces so it has no choice
214 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
216 # Store the Python as a simple scalar
217 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
219 def _yaml_config_constructor(loader, node):
220 """Construct a config from YAML."""
221 config_py = loader.construct_scalar(node)
222 return Config._fromPython(config_py)
224 # Register a generic constructor for Config and all subclasses
225 # Need to register for all the loaders we would like to use
226 for loader in YamlLoaders:
227 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
230class ConfigMeta(type):
231 """A metaclass for `lsst.pex.config.Config`.
233 Parameters
234 ----------
235 name : `str`
236 Name to use for class.
237 bases : `~collections.abc.Iterable`
238 Base classes.
239 dict_ : `dict`
240 Additional parameters.
242 Notes
243 -----
244 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
245 class attributes as a class attribute called ``_fields``, and adds
246 the name of each field as an instance variable of the field itself (so you
247 don't have to pass the name of the field to the field constructor).
248 """
250 def __init__(cls, name, bases, dict_):
251 type.__init__(cls, name, bases, dict_)
252 cls._fields = {}
253 cls._source = getStackFrame()
255 def getFields(classtype):
256 fields = {}
257 bases = list(classtype.__bases__)
258 bases.reverse()
259 for b in bases:
260 fields.update(getFields(b))
262 for k, v in classtype.__dict__.items():
263 if isinstance(v, Field):
264 fields[k] = v
265 return fields
267 fields = getFields(cls)
268 for k, v in fields.items():
269 setattr(cls, k, copy.deepcopy(v))
271 def __setattr__(cls, name, value):
272 if isinstance(value, Field):
273 value.name = name
274 cls._fields[name] = value
275 type.__setattr__(cls, name, value)
278class FieldValidationError(ValueError):
279 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
280 particular ``~lsst.pex.config.Config``.
282 Parameters
283 ----------
284 field : `lsst.pex.config.Field`
285 The field that was not valid.
286 config : `lsst.pex.config.Config`
287 The config containing the invalid field.
288 msg : `str`
289 Text describing why the field was not valid.
290 """
292 def __init__(self, field, config, msg):
293 self.fieldType = type(field)
294 """Type of the `~lsst.pex.config.Field` that incurred the error.
295 """
297 self.fieldName = field.name
298 """Name of the `~lsst.pex.config.Field` instance that incurred the
299 error (`str`).
301 See also
302 --------
303 ``lsst.pex.config.Field.name``
304 """
306 self.fullname = _joinNamePath(config._name, field.name)
307 """Fully-qualified name of the `~lsst.pex.config.Field` instance
308 (`str`).
309 """
311 self.history = config.history.setdefault(field.name, [])
312 """Full history of all changes to the `~lsst.pex.config.Field`
313 instance.
314 """
316 self.fieldSource = field.source
317 """File and line number of the `~lsst.pex.config.Field` definition.
318 """
320 self.configSource = config._source
321 error = (
322 f"{self.fieldType.__name__} '{self.fullname}' failed validation: {msg}\n"
323 f"For more information see the Field definition at:\n{self.fieldSource.format()}"
324 f" and the Config definition for {_typeStr(config)} at:\n{self.configSource.format()}"
325 )
326 super().__init__(error)
329class Field(Generic[FieldTypeVar]):
330 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
331 `complex`, `bool`, and `str` data types.
333 Parameters
334 ----------
335 doc : `str`
336 A description of the field for users.
337 dtype : `type`, optional
338 The field's data type. ``Field`` only supports basic data types:
339 `int`, `float`, `complex`, `bool`, and `str`. See
340 `Field.supportedTypes`. Optional if supplied as a typing argument to
341 the class.
342 default : `object`, optional
343 The field's default value.
344 check : `collections.abc.Callable`, optional
345 A callable that is called with the field's value. This callable should
346 return `False` if the value is invalid. More complex inter-field
347 validation can be written as part of the
348 `lsst.pex.config.Config.validate` method.
349 optional : `bool`, optional
350 This sets whether the field is considered optional, and therefore
351 doesn't need to be set by the user. When `False`,
352 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
353 deprecated : `None` or `str`, optional
354 A description of why this Field is deprecated, including removal date.
355 If not None, the string is appended to the docstring for this Field.
357 Raises
358 ------
359 ValueError
360 Raised when the ``dtype`` parameter is not one of the supported types
361 (see `Field.supportedTypes`).
363 See Also
364 --------
365 ChoiceField
366 ConfigChoiceField
367 ConfigDictField
368 ConfigField
369 ConfigurableField
370 DictField
371 ListField
372 RangeField
373 RegistryField
375 Notes
376 -----
377 `Field` instances (including those of any subclass of `Field`) are used
378 as class attributes of `~lsst.pex.config.Config` subclasses (see the
379 example, below). ``Field`` attributes work like the `property` attributes
380 of classes that implement custom setters and getters. `Field` attributes
381 belong to the class, but operate on the instance. Formally speaking,
382 `Field` attributes are `descriptors
383 <https://docs.python.org/3/howto/descriptor.html>`_.
385 When you access a `Field` attribute on a `Config` instance, you don't
386 get the `Field` instance itself. Instead, you get the value of that field,
387 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
388 container type (like a `lsst.pex.config.ListField`) depending on the
389 field's type. See the example, below.
391 Fields can be annotated with a type similar to other python classes (python
392 specification `here <https://peps.python.org/pep-0484/#generics>`_ ).
393 See the name field in the Config example below for an example of this.
394 Unlike most other uses in python, this has an effect at type checking *and*
395 runtime. If the type is specified with a class annotation, it will be used
396 as the value of the ``dtype`` in the `Field` and there is no need to
397 specify it as an argument during instantiation.
399 There are Some notes on dtype through type annotation syntax. Type
400 annotation syntax supports supplying the argument as a string of a type
401 name. i.e. "float", but this cannot be used to resolve circular references.
402 Type annotation syntax can be used on an identifier in addition to Class
403 assignment i.e. ``variable: Field[str] = Config.someField`` vs
404 ``someField = Field[str](doc="some doc")``. However, this syntax is only
405 useful for annotating the type of the identifier (i.e. variable in previous
406 example) and does nothing for assigning the dtype of the `Field`.
408 Examples
409 --------
410 Instances of `Field` should be used as class attributes of
411 `lsst.pex.config.Config` subclasses:
413 >>> from lsst.pex.config import Config, Field
414 >>> class Example(Config):
415 ... myInt = Field("An integer field.", int, default=0)
416 ... name = Field[str](doc="A string Field")
417 >>> config = Example()
418 >>> print(config.myInt)
419 0
420 >>> config.myInt = 5
421 >>> print(config.myInt)
422 5
423 """
425 name: str
426 """Identifier (variable name) used to refer to a Field within a Config
427 Class.
428 """
430 supportedTypes = {str, bool, float, int, complex}
431 """Supported data types for field values (`set` of types).
432 """
434 @staticmethod
435 def _parseTypingArgs(
436 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
437 ) -> Mapping[str, Any]:
438 """Parse type annotations into keyword constructor arguments.
440 This is a special private method that interprets type arguments (i.e.
441 Field[str]) into keyword arguments to be passed on to the constructor.
443 Subclasses of Field can implement this method to customize how they
444 handle turning type parameters into keyword arguments (see DictField
445 for an example)
447 Parameters
448 ----------
449 params : `tuple` of `type` or `tuple` of str
450 Parameters passed to the type annotation. These will either be
451 types or strings. Strings are to interpreted as forward references
452 and will be treated as such.
453 kwds : `MutableMapping` with keys of `str` and values of `Any`
454 These are the user supplied keywords that are to be passed to the
455 Field constructor.
457 Returns
458 -------
459 kwds : `MutableMapping` with keys of `str` and values of `Any`
460 The mapping of keywords that will be passed onto the constructor
461 of the Field. Should be filled in with any information gleaned
462 from the input parameters.
464 Raises
465 ------
466 ValueError
467 Raised if params is of incorrect length.
468 Raised if a forward reference could not be resolved
469 Raised if there is a conflict between params and values in kwds
470 """
471 if len(params) > 1:
472 raise ValueError("Only single type parameters are supported")
473 unpackedParams = params[0]
474 if isinstance(unpackedParams, str):
475 _typ = ForwardRef(unpackedParams)
476 # type ignore below because typeshed seems to be wrong. It
477 # indicates there are only 2 args, as it was in python 3.8, but
478 # 3.9+ takes 3 args.
479 result = _typ._evaluate(globals(), locals(), recursive_guard=set()) # type: ignore
480 if result is None:
481 raise ValueError("Could not deduce type from input")
482 unpackedParams = cast(type, result)
483 if "dtype" in kwds and kwds["dtype"] != unpackedParams:
484 raise ValueError("Conflicting definition for dtype")
485 elif "dtype" not in kwds:
486 kwds = {**kwds, **{"dtype": unpackedParams}}
487 return kwds
489 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef):
490 return _PexConfigGenericAlias(cls, params)
492 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None):
493 if dtype is None: 493 ↛ 494line 493 didn't jump to line 494 because the condition on line 493 was never true
494 raise ValueError(
495 "dtype must either be supplied as an argument or as a type argument to the class"
496 )
497 if dtype not in self.supportedTypes: 497 ↛ 498line 497 didn't jump to line 498 because the condition on line 497 was never true
498 raise ValueError(f"Unsupported Field dtype {_typeStr(dtype)}")
500 source = getStackFrame()
501 self._setup(
502 doc=doc,
503 dtype=dtype,
504 default=default,
505 check=check,
506 optional=optional,
507 source=source,
508 deprecated=deprecated,
509 )
511 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
512 """Set attributes, usually during initialization."""
513 self.dtype = dtype
514 """Data type for the field.
515 """
517 if not doc: 517 ↛ 518line 517 didn't jump to line 518 because the condition on line 517 was never true
518 raise ValueError("Docstring is empty.")
520 # append the deprecation message to the docstring.
521 if deprecated is not None:
522 doc = f"{doc} Deprecated: {deprecated}"
523 self.doc = doc
524 """A description of the field (`str`).
525 """
527 self.deprecated = deprecated
528 """If not None, a description of why this field is deprecated (`str`).
529 """
531 self.__doc__ = f"{doc} (`{dtype.__name__}`"
532 if optional or default is not None:
533 self.__doc__ += f", default ``{default!r}``"
534 self.__doc__ += ")"
536 self.default = default
537 """Default value for this field.
538 """
540 self.check = check
541 """A user-defined function that validates the value of the field.
542 """
544 self.optional = optional
545 """Flag that determines if the field is required to be set (`bool`).
547 When `False`, `lsst.pex.config.Config.validate` will fail if the
548 field's value is `None`.
549 """
551 self.source = source
552 """The stack frame where this field is defined (`list` of
553 `~lsst.pex.config.callStack.StackFrame`).
554 """
556 def rename(self, instance):
557 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
558 only).
560 Parameters
561 ----------
562 instance : `lsst.pex.config.Config`
563 The config instance that contains this field.
565 Notes
566 -----
567 This method is invoked by the `lsst.pex.config.Config` object that
568 contains this field and should not be called directly.
570 Renaming is only relevant for `~lsst.pex.config.Field` instances that
571 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should
572 rename each subconfig with the full field name as generated by
573 `lsst.pex.config.config._joinNamePath`.
574 """
575 pass
577 def validate(self, instance):
578 """Validate the field (for internal use only).
580 Parameters
581 ----------
582 instance : `lsst.pex.config.Config`
583 The config instance that contains this field.
585 Raises
586 ------
587 lsst.pex.config.FieldValidationError
588 Raised if verification fails.
590 Notes
591 -----
592 This method provides basic validation:
594 - Ensures that the value is not `None` if the field is not optional.
595 - Ensures type correctness.
596 - Ensures that the user-provided ``check`` function is valid.
598 Most `~lsst.pex.config.Field` subclasses should call
599 `lsst.pex.config.Field.validate` if they re-implement
600 `~lsst.pex.config.Field.validate`.
601 """
602 value = self.__get__(instance)
603 if not self.optional and value is None:
604 raise FieldValidationError(self, instance, "Required value cannot be None")
606 def freeze(self, instance):
607 """Make this field read-only (for internal use only).
609 Parameters
610 ----------
611 instance : `lsst.pex.config.Config`
612 The config instance that contains this field.
614 Notes
615 -----
616 Freezing is only relevant for fields that hold subconfigs. Fields which
617 hold subconfigs should freeze each subconfig.
619 **Subclasses should implement this method.**
620 """
621 pass
623 def _validateValue(self, value):
624 """Validate a value.
626 Parameters
627 ----------
628 value : `object`
629 The value being validated.
631 Raises
632 ------
633 TypeError
634 Raised if the value's type is incompatible with the field's
635 ``dtype``.
636 ValueError
637 Raised if the value is rejected by the ``check`` method.
638 """
639 if value is None: 639 ↛ 640line 639 didn't jump to line 640 because the condition on line 639 was never true
640 return
642 if not isinstance(value, self.dtype): 642 ↛ 643line 642 didn't jump to line 643 because the condition on line 642 was never true
643 msg = (
644 f"Value {value} is of incorrect type {_typeStr(value)}. Expected type {_typeStr(self.dtype)}"
645 )
646 raise TypeError(msg)
647 if self.check is not None and not self.check(value): 647 ↛ 648line 647 didn't jump to line 648 because the condition on line 647 was never true
648 msg = f"Value {value} is not a valid value"
649 raise ValueError(msg)
651 def _collectImports(self, instance, imports):
652 """Call the _collectImports method on all config
653 objects the field may own, and union them with the supplied imports
654 set.
656 Parameters
657 ----------
658 instance : instance or subclass of `lsst.pex.config.Config`
659 A config object that has this field defined on it
660 imports : `set`
661 Set of python modules that need imported after persistence
662 """
663 pass
665 def save(self, outfile, instance):
666 """Save this field to a file (for internal use only).
668 Parameters
669 ----------
670 outfile : `typing.IO`
671 A writeable file handle.
672 instance : `~lsst.pex.config.Config`
673 The `~lsst.pex.config.Config` instance that contains this field.
675 Notes
676 -----
677 This method is invoked by the `~lsst.pex.config.Config` object that
678 contains this field and should not be called directly.
680 The output consists of the documentation string
681 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
682 line is formatted as an assignment: ``{fullname}={value}``.
684 This output can be executed with Python.
685 """
686 value = self.__get__(instance)
687 fullname = _joinNamePath(instance._name, self.name)
689 if self.deprecated and value == self.default: 689 ↛ 690line 689 didn't jump to line 690 because the condition on line 689 was never true
690 return
692 # write full documentation string as comment lines
693 # (i.e. first character is #)
694 doc = "# " + str(self.doc).replace("\n", "\n# ")
695 if isinstance(value, float) and not math.isfinite(value): 695 ↛ 697line 695 didn't jump to line 697 because the condition on line 695 was never true
696 # non-finite numbers need special care
697 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n")
698 else:
699 outfile.write(f"{doc}\n{fullname}={value!r}\n\n")
701 def toDict(self, instance):
702 """Convert the field value so that it can be set as the value of an
703 item in a `dict` (for internal use only).
705 Parameters
706 ----------
707 instance : `~lsst.pex.config.Config`
708 The `~lsst.pex.config.Config` that contains this field.
710 Returns
711 -------
712 value : `object`
713 The field's value. See *Notes*.
715 Notes
716 -----
717 This method invoked by the owning `~lsst.pex.config.Config` object and
718 should not be called directly.
720 Simple values are passed through. Complex data structures must be
721 manipulated. For example, a `~lsst.pex.config.Field` holding a
722 subconfig should, instead of the subconfig object, return a `dict`
723 where the keys are the field names in the subconfig, and the values are
724 the field values in the subconfig.
725 """
726 return self.__get__(instance)
728 def _copy_storage(self, old: Config, new: Config) -> Any:
729 """Copy the storage for this field in the given field into an object
730 suitable for storage in a new copy of that config.
732 Any frozen storage should be unfrozen.
733 """
734 return copy.deepcopy(old._storage[self.name])
736 @overload
737 def __get__(
738 self, instance: None, owner: Any = None, at: Any = None, label: str = "default"
739 ) -> Field[FieldTypeVar]: ...
741 @overload
742 def __get__(
743 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default"
744 ) -> FieldTypeVar: ...
746 def __get__(self, instance, owner=None, at=None, label="default"):
747 """Define how attribute access should occur on the Config instance
748 This is invoked by the owning config object and should not be called
749 directly.
751 When the field attribute is accessed on a Config class object, it
752 returns the field object itself in order to allow inspection of
753 Config classes.
755 When the field attribute is access on a config instance, the actual
756 value described by the field (and held by the Config instance) is
757 returned.
758 """
759 if instance is None: 759 ↛ 760line 759 didn't jump to line 760 because the condition on line 759 was never true
760 return self
761 else:
762 # try statements are almost free in python if they succeed
763 try:
764 return instance._storage[self.name]
765 except AttributeError:
766 if not isinstance(instance, Config):
767 return self
768 else:
769 raise AttributeError(
770 f"Config {instance} is missing _storage attribute, likely incorrectly initialized"
771 ) from None
773 def __set__(
774 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment"
775 ) -> None:
776 """Set an attribute on the config instance.
778 Parameters
779 ----------
780 instance : `lsst.pex.config.Config`
781 The config instance that contains this field.
782 value : obj
783 Value to set on this field.
784 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\
785 optional
786 The call stack (created by
787 `lsst.pex.config.callStack.getCallStack`).
788 label : `str`, optional
789 Event label for the history.
791 Notes
792 -----
793 This method is invoked by the owning `lsst.pex.config.Config` object
794 and should not be called directly.
796 Derived `~lsst.pex.config.Field` classes may need to override the
797 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
798 should follow the following rules:
800 - Do not allow modification of frozen configs.
801 - Validate the new value **before** modifying the field. Except if the
802 new value is `None`. `None` is special and no attempt should be made
803 to validate it until `lsst.pex.config.Config.validate` is called.
804 - Do not modify the `~lsst.pex.config.Config` instance to contain
805 invalid values.
806 - If the field is modified, update the history of the
807 `lsst.pex.config.field.Field` to reflect the changes.
809 In order to decrease the need to implement this method in derived
810 `~lsst.pex.config.Field` types, value validation is performed in the
811 `lsst.pex.config.Field._validateValue`. If only the validation step
812 differs in the derived `~lsst.pex.config.Field`, it is simpler to
813 implement `lsst.pex.config.Field._validateValue` than to reimplement
814 ``__set__``. More complicated behavior, however, may require
815 reimplementation.
816 """
817 if instance._frozen: 817 ↛ 818line 817 didn't jump to line 818 because the condition on line 817 was never true
818 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
820 history = instance._history.setdefault(self.name, [])
821 if value is not None: 821 ↛ 828line 821 didn't jump to line 828 because the condition on line 821 was always true
822 value = _autocast(value, self.dtype)
823 try:
824 self._validateValue(value)
825 except BaseException as e:
826 raise FieldValidationError(self, instance, str(e)) from e
828 instance._storage[self.name] = value
829 if at is None: 829 ↛ 830line 829 didn't jump to line 830 because the condition on line 829 was never true
830 at = getCallStack()
831 history.append((value, at, label))
833 def __delete__(self, instance, at=None, label="deletion"):
834 """Delete an attribute from a `lsst.pex.config.Config` instance.
836 Parameters
837 ----------
838 instance : `lsst.pex.config.Config`
839 The config instance that contains this field.
840 at : `list` of `lsst.pex.config.callStack.StackFrame`
841 The call stack (created by
842 `lsst.pex.config.callStack.getCallStack`).
843 label : `str`, optional
844 Event label for the history.
846 Notes
847 -----
848 This is invoked by the owning `~lsst.pex.config.Config` object and
849 should not be called directly.
850 """
851 if at is None:
852 at = getCallStack()
853 self.__set__(instance, None, at=at, label=label)
855 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
856 """Compare a field (named `Field.name`) in two
857 `~lsst.pex.config.Config` instances for equality.
859 Parameters
860 ----------
861 instance1 : `lsst.pex.config.Config`
862 Left-hand side `Config` instance to compare.
863 instance2 : `lsst.pex.config.Config`
864 Right-hand side `Config` instance to compare.
865 shortcut : `bool`, optional
866 **Unused.**
867 rtol : `float`, optional
868 Relative tolerance for floating point comparisons.
869 atol : `float`, optional
870 Absolute tolerance for floating point comparisons.
871 output : `collections.abc.Callable`, optional
872 A callable that takes a string, used (possibly repeatedly) to
873 report inequalities.
875 Notes
876 -----
877 This method must be overridden by more complex `Field` subclasses.
879 See Also
880 --------
881 lsst.pex.config.compareScalars
882 """
883 v1 = getattr(instance1, self.name)
884 v2 = getattr(instance2, self.name)
885 name = getComparisonName(
886 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name)
887 )
888 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
891class RecordingImporter:
892 """Importer (for `sys.meta_path`) that records which modules are being
893 imported.
895 *This class does not do any importing itself.*
897 Examples
898 --------
899 Use this class as a context manager to ensure it is properly uninstalled
900 when done:
902 >>> with RecordingImporter() as importer:
903 ... # import stuff
904 ... import numpy as np
905 ... print("Imported: " + importer.getModules())
906 """
908 def __init__(self):
909 self._modules = set()
911 def __enter__(self):
912 self.origMetaPath = sys.meta_path
913 sys.meta_path = [self] + sys.meta_path # type: ignore
914 return self
916 def __exit__(self, *args):
917 self.uninstall()
918 return False # Don't suppress exceptions
920 def uninstall(self):
921 """Uninstall the importer."""
922 sys.meta_path = self.origMetaPath
924 def find_spec(self, fullname, path, target=None):
925 """Find a module.
927 Called as part of the ``import`` chain of events.
929 Parameters
930 ----------
931 fullname : `str`
932 Name of module.
933 path : `list` [`str`]
934 Search path. Unused.
935 target : `~typing.Any`, optional
936 Unused.
937 """
938 self._modules.add(fullname)
939 # Return None because we don't do any importing.
940 return None
942 def getModules(self):
943 """Get the set of modules that were imported.
945 Returns
946 -------
947 modules : `set` of `str`
948 Set of imported module names.
949 """
950 return self._modules
953# type ignore because type checker thinks ConfigMeta is Generic when it is not
954class Config(metaclass=ConfigMeta): # type: ignore
955 """Base class for configuration (*config*) objects.
957 Notes
958 -----
959 A ``Config`` object will usually have several `~lsst.pex.config.Field`
960 instances as class attributes. These are used to define most of the base
961 class behavior.
963 ``Config`` implements a mapping API that provides many `dict`-like methods,
964 such as `keys`, `values`, and `items`. ``Config`` instances also support
965 the ``in`` operator to test if a field is in the config. Unlike a `dict`,
966 ``Config`` classes are not subscriptable. Instead, access individual
967 fields as attributes of the configuration instance.
969 Examples
970 --------
971 Config classes are subclasses of ``Config`` that have
972 `~lsst.pex.config.Field` instances (or instances of
973 `~lsst.pex.config.Field` subclasses) as class attributes:
975 >>> from lsst.pex.config import Config, Field, ListField
976 >>> class DemoConfig(Config):
977 ... intField = Field(doc="An integer field", dtype=int, default=42)
978 ... listField = ListField(
979 ... doc="List of favorite beverages.",
980 ... dtype=str,
981 ... default=["coffee", "green tea", "water"],
982 ... )
983 >>> config = DemoConfig()
985 Configs support many `dict`-like APIs:
987 >>> config.keys()
988 ['intField', 'listField']
989 >>> "intField" in config
990 True
992 Individual fields can be accessed as attributes of the configuration:
994 >>> config.intField
995 42
996 >>> config.listField.append("earl grey tea")
997 >>> print(config.listField)
998 ['coffee', 'green tea', 'water', 'earl grey tea']
999 """
1001 _storage: dict[str, Any]
1002 _fields: dict[str, Field]
1003 _history: dict[str, list[Any]]
1004 _imports: set[Any]
1006 def __iter__(self):
1007 """Iterate over fields."""
1008 return self._fields.__iter__()
1010 def keys(self):
1011 """Get field names.
1013 Returns
1014 -------
1015 names : `~collections.abc.KeysView`
1016 List of `lsst.pex.config.Field` names.
1017 """
1018 return self._storage.keys()
1020 def values(self):
1021 """Get field values.
1023 Returns
1024 -------
1025 values : `~collections.abc.ValuesView`
1026 Iterator of field values.
1027 """
1028 return self._storage.values()
1030 def items(self):
1031 """Get configurations as ``(field name, field value)`` pairs.
1033 Returns
1034 -------
1035 items : `~collections.abc.ItemsView`
1036 Iterator of tuples for each configuration. Tuple items are:
1038 0. Field name.
1039 1. Field value.
1040 """
1041 return self._storage.items()
1043 def __contains__(self, name):
1044 """Return `True` if the specified field exists in this config.
1046 Parameters
1047 ----------
1048 name : `str`
1049 Field name to test for.
1051 Returns
1052 -------
1053 in : `bool`
1054 `True` if the specified field exists in the config.
1055 """
1056 return self._storage.__contains__(name)
1058 def __new__(cls, *args, **kw):
1059 """Allocate a new `lsst.pex.config.Config` object.
1061 In order to ensure that all Config object are always in a proper state
1062 when handed to users or to derived `~lsst.pex.config.Config` classes,
1063 some attributes are handled at allocation time rather than at
1064 initialization.
1066 This ensures that even if a derived `~lsst.pex.config.Config` class
1067 implements ``__init__``, its author does not need to be concerned about
1068 when or even the base ``Config.__init__`` should be called.
1069 """
1070 name = kw.pop("__name", None)
1071 at = kw.pop("__at", getCallStack())
1072 # remove __label and ignore it
1073 kw.pop("__label", "default")
1075 instance = object.__new__(cls)
1076 instance._frozen = False
1077 instance._name = name
1078 instance._storage = {}
1079 instance._history = {}
1080 instance._imports = set()
1081 # load up defaults
1082 for field in instance._fields.values():
1083 instance._history[field.name] = []
1084 field.__set__(instance, field.default, at=at + [field.source], label="default")
1085 # set custom default-overrides
1086 instance.setDefaults()
1087 # set constructor overrides
1088 instance.update(__at=at, **kw)
1089 return instance
1091 def copy(self) -> Config:
1092 """Return a deep copy of this config.
1094 Notes
1095 -----
1096 The returned config object is not frozen, even if the original was.
1097 If a nested config object is copied, it retains the name from its
1098 original hierarchy.
1100 Nested objects are only shared between the new and old configs if they
1101 are not possible to modify via the config's interfaces (e.g. entries
1102 in the the history list are not copied, but the lists themselves are,
1103 so modifications to one copy do not modify the other).
1104 """
1105 instance = object.__new__(type(self))
1106 instance._frozen = False
1107 instance._name = self._name
1108 instance._history = {k: list(v) for k, v in self._history.items()}
1109 instance._imports = set(self._imports)
1110 # Important to set up storage last, since fields sometimes store
1111 # proxy objects that reference their parent (especially for history).
1112 instance._storage = {k: self._fields[k]._copy_storage(self, instance) for k in self._storage}
1113 return instance
1115 def __reduce__(self):
1116 """Reduction for pickling (function with arguments to reproduce).
1118 We need to condense and reconstitute the `~lsst.pex.config.Config`,
1119 since it may contain lambdas (as the ``check`` elements) that cannot
1120 be pickled.
1121 """
1122 # The stream must be in characters to match the API but pickle
1123 # requires bytes
1124 stream = io.StringIO()
1125 self.saveToStream(stream)
1126 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
1128 def setDefaults(self):
1129 """Subclass hook for computing defaults.
1131 Notes
1132 -----
1133 Derived `~lsst.pex.config.Config` classes that must compute defaults
1134 rather than using the `~lsst.pex.config.Field` instances's defaults
1135 should do so here. To correctly use inherited defaults,
1136 implementations of ``setDefaults`` must call their base class's
1137 ``setDefaults``.
1138 """
1139 pass
1141 def update(self, **kw):
1142 """Update values of fields specified by the keyword arguments.
1144 Parameters
1145 ----------
1146 **kw
1147 Keywords are configuration field names. Values are configuration
1148 field values.
1150 Notes
1151 -----
1152 The ``__at`` and ``__label`` keyword arguments are special internal
1153 keywords. They are used to strip out any internal steps from the
1154 history tracebacks of the config. Do not modify these keywords to
1155 subvert a `~lsst.pex.config.Config` instance's history.
1157 Examples
1158 --------
1159 This is a config with three fields:
1161 >>> from lsst.pex.config import Config, Field
1162 >>> class DemoConfig(Config):
1163 ... fieldA = Field(doc="Field A", dtype=int, default=42)
1164 ... fieldB = Field(doc="Field B", dtype=bool, default=True)
1165 ... fieldC = Field(doc="Field C", dtype=str, default="Hello world")
1166 >>> config = DemoConfig()
1168 These are the default values of each field:
1170 >>> for name, value in config.iteritems():
1171 ... print(f"{name}: {value}")
1172 fieldA: 42
1173 fieldB: True
1174 fieldC: 'Hello world'
1176 Using this method to update ``fieldA`` and ``fieldC``:
1178 >>> config.update(fieldA=13, fieldC="Updated!")
1180 Now the values of each field are:
1182 >>> for name, value in config.iteritems():
1183 ... print(f"{name}: {value}")
1184 fieldA: 13
1185 fieldB: True
1186 fieldC: 'Updated!'
1187 """
1188 at = kw.pop("__at", getCallStack())
1189 label = kw.pop("__label", "update")
1191 for name, value in kw.items():
1192 try:
1193 field = self._fields[name]
1194 field.__set__(self, value, at=at, label=label)
1195 except KeyError as e:
1196 e.add_note(f"No field of name {name} exists in config type {_typeStr(self)}")
1197 raise
1199 def _filename_to_resource(
1200 self, filename: ResourcePathExpression | None = None
1201 ) -> tuple[ResourcePath | None, str]:
1202 """Create resource path from filename.
1204 Parameters
1205 ----------
1206 filename : `lsst.resources.ResourcePathExpression` or `None`
1207 The URI expression associated with this config. Can be `None`
1208 if no file URI is known.
1210 Returns
1211 -------
1212 resource : `lsst.resources.ResourcePath` or `None`
1213 The resource version of the filename. Returns `None` if no filename
1214 was given or refers to unspecified value.
1215 file_string : `str`
1216 String form of the resource for use in ``__file__``
1217 """
1218 if filename is None or filename in ("?", "<unknown>"): 1218 ↛ 1220line 1218 didn't jump to line 1220 because the condition on line 1218 was always true
1219 return None, "<unknown>"
1220 base = _get_config_root()
1221 resource = ResourcePath(filename, forceAbsolute=True, forceDirectory=False, root=base)
1223 # Preferred definition of __file__ is the full OS path. If a config
1224 # is loaded with a relative path it must be converted to the absolute
1225 # path to avoid confusion with later relative paths referenced inside
1226 # the config.
1227 if resource.scheme == "file":
1228 file_string = resource.ospath
1229 else:
1230 file_string = str(resource)
1232 return resource, file_string
1234 def load(self, filename, root="config"):
1235 """Modify this config in place by executing the Python code in a
1236 configuration file.
1238 Parameters
1239 ----------
1240 filename : `lsst.resources.ResourcePathExpression`
1241 Name of the configuration URI. A configuration file is a Python
1242 module. Since configuration files are Python code, remote URIs
1243 are not allowed.
1244 root : `str`, optional
1245 Name of the variable in file that refers to the config being
1246 overridden.
1248 For example, the value of root is ``"config"`` and the file
1249 contains::
1251 config.myField = 5
1253 Then this config's field ``myField`` is set to ``5``.
1255 See Also
1256 --------
1257 lsst.pex.config.Config.loadFromStream
1258 lsst.pex.config.Config.loadFromString
1259 lsst.pex.config.Config.save
1260 lsst.pex.config.Config.saveToStream
1261 lsst.pex.config.Config.saveToString
1262 """
1263 resource, file_string = self._filename_to_resource(filename)
1264 if resource is None:
1265 # A filename is required.
1266 raise ValueError(f"Undefined URI provided to load command: {filename}.")
1268 if resource.scheme not in ("file", "eups", "resource"):
1269 raise ValueError(f"Remote URI ({resource}) can not be used to load configurations.")
1271 # Push the directory of the file we are now reading onto the stack
1272 # so that nested loads are relative to this file.
1273 with _push_config_root(resource.dirname()):
1274 _LOG.debug("Updating config from URI %s", str(resource))
1275 with resource.open("r") as f:
1276 code = compile(f.read(), filename=file_string, mode="exec")
1277 self._loadFromString(code, root=root, filename=file_string)
1279 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None):
1280 """Modify this Config in place by executing the Python code in the
1281 provided stream.
1283 Parameters
1284 ----------
1285 stream : `typing.IO`, `str`, `bytes`, or `~types.CodeType`
1286 Stream containing configuration override code. If this is a
1287 code object, it should be compiled with ``mode="exec"``.
1288 root : `str`, optional
1289 Name of the variable in file that refers to the config being
1290 overridden.
1292 For example, the value of root is ``"config"`` and the file
1293 contains::
1295 config.myField = 5
1297 Then this config's field ``myField`` is set to ``5``.
1298 filename : `str`, optional
1299 Name of the configuration file, or `None` if unknown or contained
1300 in the stream. Used for error reporting and to set ``__file__``
1301 variable in config.
1302 extraLocals : `dict` of `str` to `object`, optional
1303 Any extra variables to include in local scope when loading.
1305 Notes
1306 -----
1307 For backwards compatibility reasons, this method accepts strings, bytes
1308 and code objects as well as file-like objects. New code should use
1309 `loadFromString` instead for most of these types.
1311 See Also
1312 --------
1313 lsst.pex.config.Config.load
1314 lsst.pex.config.Config.loadFromString
1315 lsst.pex.config.Config.save
1316 lsst.pex.config.Config.saveToStream
1317 lsst.pex.config.Config.saveToString
1318 """
1319 if hasattr(stream, "read"): 1319 ↛ 1320line 1319 didn't jump to line 1320 because the condition on line 1319 was never true
1320 if filename is None:
1321 filename = getattr(stream, "name", "<unknown>")
1322 code = compile(stream.read(), filename=filename, mode="exec")
1323 else:
1324 code = stream
1325 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1327 def loadFromString(self, code, root="config", filename=None, extraLocals=None):
1328 """Modify this Config in place by executing the Python code in the
1329 provided string.
1331 Parameters
1332 ----------
1333 code : `str`, `bytes`, or `~types.CodeType`
1334 Stream containing configuration override code.
1335 root : `str`, optional
1336 Name of the variable in file that refers to the config being
1337 overridden.
1339 For example, the value of root is ``"config"`` and the file
1340 contains::
1342 config.myField = 5
1344 Then this config's field ``myField`` is set to ``5``.
1345 filename : `lsst.resources.ResourcePathExpression`, optional
1346 URI of the configuration file, or `None` if unknown or contained
1347 in the stream. Used for error reporting and to set ``__file__``
1348 variable. Required to be set if the string config attempts to
1349 load other configs using either relative path or ``__file__``.
1350 extraLocals : `dict` of `str` to `object`, optional
1351 Any extra variables to include in local scope when loading.
1353 Raises
1354 ------
1355 ValueError
1356 Raised if a key in extraLocals is the same value as the value of
1357 the root argument.
1359 See Also
1360 --------
1361 lsst.pex.config.Config.load
1362 lsst.pex.config.Config.loadFromStream
1363 lsst.pex.config.Config.save
1364 lsst.pex.config.Config.saveToStream
1365 lsst.pex.config.Config.saveToString
1366 """
1367 if filename is None: 1367 ↛ 1372line 1367 didn't jump to line 1372 because the condition on line 1367 was always true
1368 # try to determine the file name; a compiled string
1369 # has attribute "co_filename",
1370 filename = getattr(code, "co_filename", "<unknown>")
1372 resource, file_string = self._filename_to_resource(filename)
1373 if resource is None: 1373 ↛ 1380line 1373 didn't jump to line 1380 because the condition on line 1373 was always true
1374 # No idea where this config came from so no ability to deal with
1375 # relative paths. No reason to use context.
1376 self._loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1377 else:
1378 # Push the directory of the file we are now reading onto the stack
1379 # so that nested loads are relative to this file.
1380 with _push_config_root(resource.dirname()):
1381 self._loadFromString(code, root=root, filename=file_string, extraLocals=extraLocals)
1383 def _loadFromString(self, code, root="config", filename=None, extraLocals=None):
1384 """Update config from string.
1386 Assumes relative directory path context has been setup by caller.
1387 """
1388 with RecordingImporter() as importer:
1389 globals = {"__file__": filename}
1390 local = {root: self}
1391 if extraLocals is not None: 1391 ↛ 1393line 1391 didn't jump to line 1393 because the condition on line 1391 was never true
1392 # verify the value of root was not passed as extra local args
1393 if root in extraLocals:
1394 raise ValueError(
1395 f"{root} is reserved and cannot be used as a variable name in extraLocals"
1396 )
1397 local.update(extraLocals)
1398 exec(code, globals, local)
1400 self._imports.update(importer.getModules())
1402 def save(self, filename, root="config"):
1403 """Save a Python script to the named file, which, when loaded,
1404 reproduces this config.
1406 Parameters
1407 ----------
1408 filename : `str`
1409 Desination filename of this configuration.
1410 root : `str`, optional
1411 Name to use for the root config variable. The same value must be
1412 used when loading (see `lsst.pex.config.Config.load`).
1414 See Also
1415 --------
1416 lsst.pex.config.Config.saveToStream
1417 lsst.pex.config.Config.saveToString
1418 lsst.pex.config.Config.load
1419 lsst.pex.config.Config.loadFromStream
1420 lsst.pex.config.Config.loadFromString
1421 """
1422 d = os.path.dirname(filename)
1423 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1424 self.saveToStream(outfile, root)
1425 # tempfile is hardcoded to create files with mode '0600'
1426 # for an explantion of these antics see:
1427 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1428 umask = os.umask(0o077)
1429 os.umask(umask)
1430 os.chmod(outfile.name, (~umask & 0o666))
1431 # chmod before the move so we get quasi-atomic behavior if the
1432 # source and dest. are on the same filesystem.
1433 # os.rename may not work across filesystems
1434 shutil.move(outfile.name, filename)
1436 def saveToString(self, skipImports=False):
1437 """Return the Python script form of this configuration as an executable
1438 string.
1440 Parameters
1441 ----------
1442 skipImports : `bool`, optional
1443 If `True` then do not include ``import`` statements in output,
1444 this is to support human-oriented output from ``pipetask`` where
1445 additional clutter is not useful.
1447 Returns
1448 -------
1449 code : `str`
1450 A code string readable by `loadFromString`.
1452 See Also
1453 --------
1454 lsst.pex.config.Config.save
1455 lsst.pex.config.Config.saveToStream
1456 lsst.pex.config.Config.load
1457 lsst.pex.config.Config.loadFromStream
1458 lsst.pex.config.Config.loadFromString
1459 """
1460 buffer = io.StringIO()
1461 self.saveToStream(buffer, skipImports=skipImports)
1462 return buffer.getvalue()
1464 def saveToStream(self, outfile, root="config", skipImports=False):
1465 """Save a configuration file to a stream, which, when loaded,
1466 reproduces this config.
1468 Parameters
1469 ----------
1470 outfile : `typing.TextIO`
1471 Destination file object write the config into. Accepts strings not
1472 bytes.
1473 root : `str`, optional
1474 Name to use for the root config variable. The same value must be
1475 used when loading (see `lsst.pex.config.Config.load`).
1476 skipImports : `bool`, optional
1477 If `True` then do not include ``import`` statements in output,
1478 this is to support human-oriented output from ``pipetask`` where
1479 additional clutter is not useful.
1481 See Also
1482 --------
1483 lsst.pex.config.Config.save
1484 lsst.pex.config.Config.saveToString
1485 lsst.pex.config.Config.load
1486 lsst.pex.config.Config.loadFromStream
1487 lsst.pex.config.Config.loadFromString
1488 """
1489 tmp = self._name
1490 self._rename(root)
1491 try:
1492 if not skipImports: 1492 ↛ 1509line 1492 didn't jump to line 1509 because the condition on line 1492 was always true
1493 self._collectImports()
1494 # Remove self from the set, as it is handled explicitly below
1495 self._imports.remove(self.__module__)
1496 configType = type(self)
1497 typeString = _typeStr(configType)
1498 outfile.write(f"import {configType.__module__}\n")
1499 # We are required to write this on a single line because
1500 # of later regex matching, rather than adopting black style
1501 # formatting.
1502 outfile.write(
1503 f'assert type({root}) is {typeString}, f"config is of type '
1504 f'{{type({root}).__module__}}.{{type({root}).__name__}} instead of {typeString}"\n\n'
1505 )
1506 for imp in sorted(self._imports): 1506 ↛ 1507line 1506 didn't jump to line 1507 because the loop on line 1506 never started
1507 if imp in sys.modules and sys.modules[imp] is not None:
1508 outfile.write(f"import {imp}\n")
1509 self._save(outfile)
1510 finally:
1511 self._rename(tmp)
1513 def freeze(self):
1514 """Make this config, and all subconfigs, read-only."""
1515 self._frozen = True
1516 for field in self._fields.values():
1517 field.freeze(self)
1519 def _save(self, outfile):
1520 """Save this config to an open stream object.
1522 Parameters
1523 ----------
1524 outfile : `typing.TextIO`
1525 Destination file object write the config into. Accepts strings not
1526 bytes.
1527 """
1528 for field in self._fields.values():
1529 field.save(outfile, self)
1531 def _collectImports(self):
1532 """Add module containing self to the list of things to import and
1533 then loops over all the fields in the config calling a corresponding
1534 collect method.
1536 The field method will call _collectImports on any
1537 configs it may own and return the set of things to import. This
1538 returned set will be merged with the set of imports for this config
1539 class.
1540 """
1541 self._imports.add(self.__module__)
1542 for field in self._fields.values():
1543 field._collectImports(self, self._imports)
1545 def toDict(self):
1546 """Make a dictionary of field names and their values.
1548 Returns
1549 -------
1550 dict_ : `dict`
1551 Dictionary with keys that are `~lsst.pex.config.Field` names.
1552 Values are `~lsst.pex.config.Field` values.
1554 See Also
1555 --------
1556 lsst.pex.config.Field.toDict
1558 Notes
1559 -----
1560 This method uses the `~lsst.pex.config.Field.toDict` method of
1561 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1562 implement a ``toDict`` method for *this* method to work.
1563 """
1564 dict_ = {}
1565 for name, field in self._fields.items():
1566 dict_[name] = field.toDict(self)
1567 return dict_
1569 def names(self):
1570 """Get all the field names in the config, recursively.
1572 Returns
1573 -------
1574 names : `list` of `str`
1575 Field names.
1576 """
1577 #
1578 # Rather than sort out the recursion all over again use the
1579 # pre-existing saveToStream()
1580 #
1581 with io.StringIO() as strFd:
1582 self.saveToStream(strFd, "config")
1583 contents = strFd.getvalue()
1584 strFd.close()
1585 #
1586 # Pull the names out of the dumped config
1587 #
1588 keys = []
1589 for line in contents.split("\n"):
1590 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1591 continue
1593 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1594 if mat:
1595 keys.append(mat.group(1))
1597 return keys
1599 def _rename(self, name):
1600 """Rename this config object in its parent `~lsst.pex.config.Config`.
1602 Parameters
1603 ----------
1604 name : `str`
1605 New name for this config in its parent `~lsst.pex.config.Config`.
1607 Notes
1608 -----
1609 This method uses the `~lsst.pex.config.Field.rename` method of
1610 individual `lsst.pex.config.Field` instances.
1611 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1612 method for *this* method to work.
1614 See Also
1615 --------
1616 lsst.pex.config.Field.rename
1617 """
1618 self._name = name
1619 for field in self._fields.values():
1620 field.rename(self)
1622 def validate(self):
1623 """Validate the Config, raising an exception if invalid.
1625 Raises
1626 ------
1627 lsst.pex.config.FieldValidationError
1628 Raised if verification fails.
1630 Notes
1631 -----
1632 The base class implementation performs type checks on all fields by
1633 calling their `~lsst.pex.config.Field.validate` methods.
1635 Complex single-field validation can be defined by deriving new Field
1636 types. For convenience, some derived `lsst.pex.config.Field`-types
1637 (`~lsst.pex.config.ConfigField` and
1638 `~lsst.pex.config.ConfigChoiceField`) are defined in
1639 ``lsst.pex.config`` that handle recursing into subconfigs.
1641 Inter-field relationships should only be checked in derived
1642 `~lsst.pex.config.Config` classes after calling this method, and base
1643 validation is complete.
1644 """
1645 for field in self._fields.values():
1646 field.validate(self)
1648 def formatHistory(self, name, **kwargs):
1649 """Format a configuration field's history to a human-readable format.
1651 Parameters
1652 ----------
1653 name : `str`
1654 Name of a `~lsst.pex.config.Field` in this config.
1655 **kwargs
1656 Keyword arguments passed to `lsst.pex.config.history.format`.
1658 Returns
1659 -------
1660 history : `str`
1661 A string containing the formatted history.
1663 See Also
1664 --------
1665 lsst.pex.config.history.format
1666 """
1667 import lsst.pex.config.history as pexHist
1669 return pexHist.format(self, name, **kwargs)
1671 history = property(lambda x: x._history)
1672 """Read-only history.
1673 """
1675 def __setattr__(self, attr, value, at=None, label="assignment"):
1676 """Set an attribute (such as a field's value).
1678 Notes
1679 -----
1680 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1681 locked such that no additional attributes nor properties may be added
1682 to them dynamically.
1684 Although this is not the standard Python behavior, it helps to protect
1685 users from accidentally mispelling a field name, or trying to set a
1686 non-existent field.
1687 """
1688 if attr in self._fields:
1689 if self._fields[attr].deprecated is not None: 1689 ↛ 1690line 1689 didn't jump to line 1690 because the condition on line 1689 was never true
1690 fullname = _joinNamePath(self._name, self._fields[attr].name)
1691 warnings.warn(
1692 f"Config field {_typeStr(type(self))}.{fullname} is deprecated: "
1693 f"{self._fields[attr].deprecated}",
1694 FutureWarning,
1695 stacklevel=2,
1696 )
1697 if at is None: 1697 ↛ 1700line 1697 didn't jump to line 1700 because the condition on line 1697 was always true
1698 at = getCallStack()
1699 # This allows Field descriptors to work.
1700 self._fields[attr].__set__(self, value, at=at, label=label)
1701 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1701 ↛ 1703line 1701 didn't jump to line 1703 because the condition on line 1701 was never true
1702 # This allows properties and other non-Field descriptors to work.
1703 return object.__setattr__(self, attr, value)
1704 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1704 ↛ 1709line 1704 didn't jump to line 1709 because the condition on line 1704 was always true
1705 # This allows specific private attributes to work.
1706 self.__dict__[attr] = value
1707 else:
1708 # We throw everything else.
1709 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
1711 def __delattr__(self, attr, at=None, label="deletion"):
1712 if attr in self._fields:
1713 if at is None:
1714 at = getCallStack()
1715 self._fields[attr].__delete__(self, at=at, label=label)
1716 else:
1717 object.__delattr__(self, attr)
1719 def __eq__(self, other):
1720 if type(other) is type(self): 1720 ↛ 1721line 1720 didn't jump to line 1721 because the condition on line 1720 was never true
1721 for name in self._fields:
1722 thisValue = getattr(self, name)
1723 otherValue = getattr(other, name)
1724 if isinstance(thisValue, float) and math.isnan(thisValue):
1725 if not math.isnan(otherValue):
1726 return False
1727 elif thisValue != otherValue:
1728 return False
1729 return True
1730 return False
1732 def __ne__(self, other):
1733 return not self.__eq__(other)
1735 def __str__(self):
1736 return str(self.toDict())
1738 def __repr__(self):
1739 return "{}({})".format(
1740 _typeStr(self),
1741 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None),
1742 )
1744 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None):
1745 """Compare this configuration to another `~lsst.pex.config.Config` for
1746 equality.
1748 Parameters
1749 ----------
1750 other : `lsst.pex.config.Config`
1751 Other `~lsst.pex.config.Config` object to compare against this
1752 config.
1753 shortcut : `bool`, optional
1754 If `True`, return as soon as an inequality is found. Default is
1755 `True`.
1756 rtol : `float`, optional
1757 Relative tolerance for floating point comparisons.
1758 atol : `float`, optional
1759 Absolute tolerance for floating point comparisons.
1760 output : `collections.abc.Callable`, optional
1761 A callable that takes a string, used (possibly repeatedly) to
1762 report inequalities.
1764 Returns
1765 -------
1766 isEqual : `bool`
1767 `True` when the two `lsst.pex.config.Config` instances are equal.
1768 `False` if there is an inequality.
1770 See Also
1771 --------
1772 lsst.pex.config.compareConfigs
1774 Notes
1775 -----
1776 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1777 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1778 are not considered by this method.
1780 Floating point comparisons are performed by `numpy.allclose`.
1781 """
1782 name1 = self._name if self._name is not None else "config"
1783 name2 = other._name if other._name is not None else "config"
1784 name = getComparisonName(name1, name2)
1785 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output)
1787 @classmethod
1788 def __init_subclass__(cls, **kwargs):
1789 """Run initialization for every subclass.
1791 Specifically registers the subclass with a YAML representer
1792 and YAML constructor (if pyyaml is available)
1793 """
1794 super().__init_subclass__(**kwargs)
1796 if not yaml: 1796 ↛ 1797line 1796 didn't jump to line 1797 because the condition on line 1796 was never true
1797 return
1799 yaml.add_representer(cls, _yaml_config_representer)
1801 @classmethod
1802 def _fromPython(cls, config_py):
1803 """Instantiate a `Config`-subclass from serialized Python form.
1805 Parameters
1806 ----------
1807 config_py : `str`
1808 A serialized form of the Config as created by
1809 `Config.saveToStream`.
1811 Returns
1812 -------
1813 config : `Config`
1814 Reconstructed `Config` instant.
1815 """
1816 cls = _classFromPython(config_py)
1817 return unreduceConfig(cls, config_py)
1820def _classFromPython(config_py):
1821 """Return the Config subclass required by this Config serialization.
1823 Parameters
1824 ----------
1825 config_py : `str`
1826 A serialized form of the Config as created by
1827 `Config.saveToStream`.
1829 Returns
1830 -------
1831 cls : `type`
1832 The `Config` subclass associated with this config.
1833 """
1834 # standard serialization has the form:
1835 # import config.class
1836 # assert type(config) is config.class.Config, ...
1837 # Older files use "type(config)==" instead.
1838 # We want to parse these two lines so we can get the class itself
1840 # Do a single regex to avoid large string copies when splitting a
1841 # large config into separate lines.
1842 # The assert regex cannot be greedy because the assert error string
1843 # can include both "," and " is ".
1844 matches = re.search(r"^import ([\w.]+)\nassert type\(\S+\)(?:\s*==\s*| is )(.*?),", config_py)
1846 if not matches:
1847 first_line, second_line, _ = config_py.split("\n", 2)
1848 raise ValueError(
1849 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}"
1850 )
1852 module_name = matches.group(1)
1853 module = importlib.import_module(module_name)
1855 # Second line
1856 full_name = matches.group(2)
1858 # Remove the module name from the full name
1859 if not full_name.startswith(module_name):
1860 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1862 # if module name is a.b.c and full name is a.b.c.d.E then
1863 # we need to remove a.b.c. and iterate over the remainder
1864 # The +1 is for the extra dot after a.b.c
1865 remainder = full_name[len(module_name) + 1 :]
1866 components = remainder.split(".")
1867 pytype = module
1868 for component in components:
1869 pytype = getattr(pytype, component)
1870 return pytype
1873def unreduceConfig(cls_, stream):
1874 """Create a `~lsst.pex.config.Config` from a stream.
1876 Parameters
1877 ----------
1878 cls_ : `lsst.pex.config.Config`-type
1879 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1880 with configurations in the ``stream``.
1881 stream : `typing.IO`, `str`, or `~types.CodeType`
1882 Stream containing configuration override code.
1884 Returns
1885 -------
1886 config : `lsst.pex.config.Config`
1887 Config instance.
1889 See Also
1890 --------
1891 lsst.pex.config.Config.loadFromStream
1892 """
1893 config = cls_()
1894 config.loadFromStream(stream)
1895 return config