Coverage for python / lsst / pex / config / config.py: 56%
500 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-20 01:28 -0800
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "Config",
31 "ConfigMeta",
32 "Field",
33 "FieldTypeVar",
34 "FieldValidationError",
35 "UnexpectedProxyUsageError",
36)
38import copy
39import importlib
40import io
41import logging
42import math
43import numbers
44import os
45import re
46import shutil
47import sys
48import tempfile
49import warnings
50from collections.abc import Mapping
51from contextlib import contextmanager
52from contextvars import ContextVar
53from types import GenericAlias
54from typing import Any, ForwardRef, Generic, TypeVar, cast, overload
56from lsst.resources import ResourcePath, ResourcePathExpression
58# if YAML is not available that's fine and we simply don't register
59# the yaml representer since we know it won't be used.
60try:
61 import yaml
62except ImportError:
63 yaml = None
65from .callStack import getCallStack, getStackFrame
66from .comparison import compareConfigs, compareScalars, getComparisonName
68if yaml: 68 ↛ 79line 68 didn't jump to line 79 because the condition on line 68 was always true
69 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
71 try:
72 # CLoader is not always available
73 from yaml import CLoader
75 YamlLoaders += (CLoader,)
76 except ImportError:
77 pass
78else:
79 YamlLoaders = ()
80 doImport = None
82_LOG = logging.getLogger(__name__)
85# Tracks the current config directory for the current context.
86_config_dir_stack: ContextVar[ResourcePath | None] = ContextVar("_config_dir_stack", default=None)
89def _get_config_root() -> ResourcePath | None:
90 return _config_dir_stack.get()
93@contextmanager
94def _push_config_root(dirname: ResourcePath):
95 token = _config_dir_stack.set(dirname)
96 try:
97 yield
98 finally:
99 _config_dir_stack.reset(token)
102class _PexConfigGenericAlias(GenericAlias):
103 """A Subclass of python's GenericAlias used in defining and instantiating
104 Generics.
106 This class differs from `types.GenericAlias` in that it calls a method
107 named _parseTypingArgs defined on Fields. This method gives Field and its
108 subclasses an opportunity to transform type parameters into class key word
109 arguments. Code authors do not need to implement any returns of this object
110 directly, and instead only need implement _parseTypingArgs, if a Field
111 subclass differs from the base class implementation.
113 This class is intended to be an implementation detail, returned from a
114 Field's `__class_getitem__` method.
115 """
117 def __call__(self, *args: Any, **kwds: Any) -> Any:
118 origin_kwargs = self._parseTypingArgs(self.__args__, kwds)
119 return super().__call__(*args, **{**kwds, **origin_kwargs})
122FieldTypeVar = TypeVar("FieldTypeVar")
125class UnexpectedProxyUsageError(TypeError):
126 """Exception raised when a proxy class is used in a context that suggests
127 it should have already been converted to the thing it proxies.
128 """
131def _joinNamePath(prefix=None, name=None, index=None):
132 """Generate nested configuration names."""
133 if not prefix and not name: 133 ↛ 134line 133 didn't jump to line 134 because the condition on line 133 was never true
134 raise ValueError("Invalid name: cannot be None")
135 elif not name: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 name = prefix
137 elif prefix and name: 137 ↛ 140line 137 didn't jump to line 140 because the condition on line 137 was always true
138 name = prefix + "." + name
140 if index is not None: 140 ↛ 141line 140 didn't jump to line 141 because the condition on line 140 was never true
141 return f"{name}[{index!r}]"
142 else:
143 return name
146def _autocast(x, dtype):
147 """Cast a value to a type, if appropriate.
149 Parameters
150 ----------
151 x : object
152 A value.
153 dtype : type
154 Data type, such as `float`, `int`, or `str`.
156 Returns
157 -------
158 values : object
159 If appropriate, the returned value is ``x`` cast to the given type
160 ``dtype``. If the cast cannot be performed the original value of
161 ``x`` is returned.
163 Notes
164 -----
165 Will convert numpy scalar types to the standard Python equivalents.
166 """
167 if dtype is float and isinstance(x, numbers.Real): 167 ↛ 169line 167 didn't jump to line 169 because the condition on line 167 was always true
168 return float(x)
169 if dtype is int and isinstance(x, numbers.Integral):
170 return int(x)
171 return x
174def _typeStr(x):
175 """Generate a fully-qualified type name.
177 Returns
178 -------
179 `str`
180 Fully-qualified type name.
182 Notes
183 -----
184 This function is used primarily for writing config files to be executed
185 later upon with the 'load' function.
186 """
187 if hasattr(x, "__module__") and hasattr(x, "__name__"):
188 xtype = x
189 else:
190 xtype = type(x)
191 if xtype.__module__ == "builtins": 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true
192 return xtype.__name__
193 else:
194 return f"{xtype.__module__}.{xtype.__name__}"
197if yaml: 197 ↛ 230line 197 didn't jump to line 230 because the condition on line 197 was always true
199 def _yaml_config_representer(dumper, data):
200 """Represent a Config object in a form suitable for YAML.
202 Stores the serialized stream as a scalar block string.
203 """
204 stream = io.StringIO()
205 data.saveToStream(stream)
206 config_py = stream.getvalue()
208 # Strip multiple newlines from the end of the config
209 # This simplifies the YAML to use | and not |+
210 config_py = config_py.rstrip() + "\n"
212 # Trailing spaces force pyyaml to use non-block form.
213 # Remove the trailing spaces so it has no choice
214 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
216 # Store the Python as a simple scalar
217 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
219 def _yaml_config_constructor(loader, node):
220 """Construct a config from YAML."""
221 config_py = loader.construct_scalar(node)
222 return Config._fromPython(config_py)
224 # Register a generic constructor for Config and all subclasses
225 # Need to register for all the loaders we would like to use
226 for loader in YamlLoaders:
227 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
230class ConfigMeta(type):
231 """A metaclass for `lsst.pex.config.Config`.
233 Parameters
234 ----------
235 name : `str`
236 Name to use for class.
237 bases : `~collections.abc.Iterable`
238 Base classes.
239 dict_ : `dict`
240 Additional parameters.
242 Notes
243 -----
244 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
245 class attributes as a class attribute called ``_fields``, and adds
246 the name of each field as an instance variable of the field itself (so you
247 don't have to pass the name of the field to the field constructor).
248 """
250 def __init__(cls, name, bases, dict_):
251 type.__init__(cls, name, bases, dict_)
252 cls._fields = {}
253 cls._source = getStackFrame()
255 def getFields(classtype):
256 fields = {}
257 bases = list(classtype.__bases__)
258 bases.reverse()
259 for b in bases:
260 fields.update(getFields(b))
262 for k, v in classtype.__dict__.items():
263 if isinstance(v, Field):
264 fields[k] = v
265 return fields
267 fields = getFields(cls)
268 for k, v in fields.items():
269 setattr(cls, k, copy.deepcopy(v))
271 def __setattr__(cls, name, value):
272 if isinstance(value, Field):
273 value.name = name
274 cls._fields[name] = value
275 type.__setattr__(cls, name, value)
278class FieldValidationError(ValueError):
279 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
280 particular ``~lsst.pex.config.Config``.
282 Parameters
283 ----------
284 field : `lsst.pex.config.Field`
285 The field that was not valid.
286 config : `lsst.pex.config.Config`
287 The config containing the invalid field.
288 msg : `str`
289 Text describing why the field was not valid.
290 """
292 def __init__(self, field, config, msg):
293 self.fieldType = type(field)
294 """Type of the `~lsst.pex.config.Field` that incurred the error.
295 """
297 self.fieldName = field.name
298 """Name of the `~lsst.pex.config.Field` instance that incurred the
299 error (`str`).
301 See also
302 --------
303 lsst.pex.config.Field.name
304 """
306 self.fullname = _joinNamePath(config._name, field.name)
307 """Fully-qualified name of the `~lsst.pex.config.Field` instance
308 (`str`).
309 """
311 self.history = config.history.setdefault(field.name, [])
312 """Full history of all changes to the `~lsst.pex.config.Field`
313 instance.
314 """
316 self.fieldSource = field.source
317 """File and line number of the `~lsst.pex.config.Field` definition.
318 """
320 self.configSource = config._source
321 error = (
322 f"{self.fieldType.__name__} '{self.fullname}' failed validation: {msg}\n"
323 f"For more information see the Field definition at:\n{self.fieldSource.format()}"
324 f" and the Config definition at:\n{self.configSource.format()}"
325 )
326 super().__init__(error)
329class Field(Generic[FieldTypeVar]):
330 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
331 `complex`, `bool`, and `str` data types.
333 Parameters
334 ----------
335 doc : `str`
336 A description of the field for users.
337 dtype : type, optional
338 The field's data type. ``Field`` only supports basic data types:
339 `int`, `float`, `complex`, `bool`, and `str`. See
340 `Field.supportedTypes`. Optional if supplied as a typing argument to
341 the class.
342 default : object, optional
343 The field's default value.
344 check : callable, optional
345 A callable that is called with the field's value. This callable should
346 return `False` if the value is invalid. More complex inter-field
347 validation can be written as part of the
348 `lsst.pex.config.Config.validate` method.
349 optional : `bool`, optional
350 This sets whether the field is considered optional, and therefore
351 doesn't need to be set by the user. When `False`,
352 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
353 deprecated : None or `str`, optional
354 A description of why this Field is deprecated, including removal date.
355 If not None, the string is appended to the docstring for this Field.
357 Raises
358 ------
359 ValueError
360 Raised when the ``dtype`` parameter is not one of the supported types
361 (see `Field.supportedTypes`).
363 See Also
364 --------
365 ChoiceField
366 ConfigChoiceField
367 ConfigDictField
368 ConfigField
369 ConfigurableField
370 DictField
371 ListField
372 RangeField
373 RegistryField
375 Notes
376 -----
377 ``Field`` instances (including those of any subclass of ``Field``) are used
378 as class attributes of `~lsst.pex.config.Config` subclasses (see the
379 example, below). ``Field`` attributes work like the `property` attributes
380 of classes that implement custom setters and getters. `Field` attributes
381 belong to the class, but operate on the instance. Formally speaking,
382 `Field` attributes are `descriptors
383 <https://docs.python.org/3/howto/descriptor.html>`_.
385 When you access a `Field` attribute on a `Config` instance, you don't
386 get the `Field` instance itself. Instead, you get the value of that field,
387 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
388 container type (like a `lsst.pex.config.List`) depending on the field's
389 type. See the example, below.
391 Fields can be annotated with a type similar to other python classes (python
392 specification `here <https://peps.python.org/pep-0484/#generics>`_ ).
393 See the name field in the Config example below for an example of this.
394 Unlike most other uses in python, this has an effect at type checking *and*
395 runtime. If the type is specified with a class annotation, it will be used
396 as the value of the ``dtype`` in the ``Field`` and there is no need to
397 specify it as an argument during instantiation.
399 There are Some notes on dtype through type annotation syntax. Type
400 annotation syntax supports supplying the argument as a string of a type
401 name. i.e. "float", but this cannot be used to resolve circular references.
402 Type annotation syntax can be used on an identifier in addition to Class
403 assignment i.e. ``variable: Field[str] = Config.someField`` vs
404 ``someField = Field[str](doc="some doc"). However, this syntax is only
405 useful for annotating the type of the identifier (i.e. variable in previous
406 example) and does nothing for assigning the dtype of the ``Field``.
408 Examples
409 --------
410 Instances of ``Field`` should be used as class attributes of
411 `lsst.pex.config.Config` subclasses:
413 >>> from lsst.pex.config import Config, Field
414 >>> class Example(Config):
415 ... myInt = Field("An integer field.", int, default=0)
416 ... name = Field[str](doc="A string Field")
417 >>> print(config.myInt)
418 0
419 >>> config.myInt = 5
420 >>> print(config.myInt)
421 5
422 """
424 name: str
425 """Identifier (variable name) used to refer to a Field within a Config
426 Class.
427 """
429 supportedTypes = {str, bool, float, int, complex}
430 """Supported data types for field values (`set` of types).
431 """
433 @staticmethod
434 def _parseTypingArgs(
435 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
436 ) -> Mapping[str, Any]:
437 """Parse type annotations into keyword constructor arguments.
439 This is a special private method that interprets type arguments (i.e.
440 Field[str]) into keyword arguments to be passed on to the constructor.
442 Subclasses of Field can implement this method to customize how they
443 handle turning type parameters into keyword arguments (see DictField
444 for an example)
446 Parameters
447 ----------
448 params : `tuple` of `type` or `tuple` of str
449 Parameters passed to the type annotation. These will either be
450 types or strings. Strings are to interpreted as forward references
451 and will be treated as such.
452 kwds : `MutableMapping` with keys of `str` and values of `Any`
453 These are the user supplied keywords that are to be passed to the
454 Field constructor.
456 Returns
457 -------
458 kwds : `MutableMapping` with keys of `str` and values of `Any`
459 The mapping of keywords that will be passed onto the constructor
460 of the Field. Should be filled in with any information gleaned
461 from the input parameters.
463 Raises
464 ------
465 ValueError
466 Raised if params is of incorrect length.
467 Raised if a forward reference could not be resolved
468 Raised if there is a conflict between params and values in kwds
469 """
470 if len(params) > 1:
471 raise ValueError("Only single type parameters are supported")
472 unpackedParams = params[0]
473 if isinstance(unpackedParams, str):
474 _typ = ForwardRef(unpackedParams)
475 # type ignore below because typeshed seems to be wrong. It
476 # indicates there are only 2 args, as it was in python 3.8, but
477 # 3.9+ takes 3 args.
478 result = _typ._evaluate(globals(), locals(), recursive_guard=set()) # type: ignore
479 if result is None:
480 raise ValueError("Could not deduce type from input")
481 unpackedParams = cast(type, result)
482 if "dtype" in kwds and kwds["dtype"] != unpackedParams:
483 raise ValueError("Conflicting definition for dtype")
484 elif "dtype" not in kwds:
485 kwds = {**kwds, **{"dtype": unpackedParams}}
486 return kwds
488 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef):
489 return _PexConfigGenericAlias(cls, params)
491 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None):
492 if dtype is None: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true
493 raise ValueError(
494 "dtype must either be supplied as an argument or as a type argument to the class"
495 )
496 if dtype not in self.supportedTypes: 496 ↛ 497line 496 didn't jump to line 497 because the condition on line 496 was never true
497 raise ValueError(f"Unsupported Field dtype {_typeStr(dtype)}")
499 source = getStackFrame()
500 self._setup(
501 doc=doc,
502 dtype=dtype,
503 default=default,
504 check=check,
505 optional=optional,
506 source=source,
507 deprecated=deprecated,
508 )
510 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
511 """Set attributes, usually during initialization."""
512 self.dtype = dtype
513 """Data type for the field.
514 """
516 if not doc: 516 ↛ 517line 516 didn't jump to line 517 because the condition on line 516 was never true
517 raise ValueError("Docstring is empty.")
519 # append the deprecation message to the docstring.
520 if deprecated is not None:
521 doc = f"{doc} Deprecated: {deprecated}"
522 self.doc = doc
523 """A description of the field (`str`).
524 """
526 self.deprecated = deprecated
527 """If not None, a description of why this field is deprecated (`str`).
528 """
530 self.__doc__ = f"{doc} (`{dtype.__name__}`"
531 if optional or default is not None:
532 self.__doc__ += f", default ``{default!r}``"
533 self.__doc__ += ")"
535 self.default = default
536 """Default value for this field.
537 """
539 self.check = check
540 """A user-defined function that validates the value of the field.
541 """
543 self.optional = optional
544 """Flag that determines if the field is required to be set (`bool`).
546 When `False`, `lsst.pex.config.Config.validate` will fail if the
547 field's value is `None`.
548 """
550 self.source = source
551 """The stack frame where this field is defined (`list` of
552 `~lsst.pex.config.callStack.StackFrame`).
553 """
555 def rename(self, instance):
556 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
557 only).
559 Parameters
560 ----------
561 instance : `lsst.pex.config.Config`
562 The config instance that contains this field.
564 Notes
565 -----
566 This method is invoked by the `lsst.pex.config.Config` object that
567 contains this field and should not be called directly.
569 Renaming is only relevant for `~lsst.pex.config.Field` instances that
570 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should
571 rename each subconfig with the full field name as generated by
572 `lsst.pex.config.config._joinNamePath`.
573 """
574 pass
576 def validate(self, instance):
577 """Validate the field (for internal use only).
579 Parameters
580 ----------
581 instance : `lsst.pex.config.Config`
582 The config instance that contains this field.
584 Raises
585 ------
586 lsst.pex.config.FieldValidationError
587 Raised if verification fails.
589 Notes
590 -----
591 This method provides basic validation:
593 - Ensures that the value is not `None` if the field is not optional.
594 - Ensures type correctness.
595 - Ensures that the user-provided ``check`` function is valid.
597 Most `~lsst.pex.config.Field` subclasses should call
598 `lsst.pex.config.Field.validate` if they re-implement
599 `~lsst.pex.config.Field.validate`.
600 """
601 value = self.__get__(instance)
602 if not self.optional and value is None:
603 raise FieldValidationError(self, instance, "Required value cannot be None")
605 def freeze(self, instance):
606 """Make this field read-only (for internal use only).
608 Parameters
609 ----------
610 instance : `lsst.pex.config.Config`
611 The config instance that contains this field.
613 Notes
614 -----
615 Freezing is only relevant for fields that hold subconfigs. Fields which
616 hold subconfigs should freeze each subconfig.
618 **Subclasses should implement this method.**
619 """
620 pass
622 def _validateValue(self, value):
623 """Validate a value.
625 Parameters
626 ----------
627 value : object
628 The value being validated.
630 Raises
631 ------
632 TypeError
633 Raised if the value's type is incompatible with the field's
634 ``dtype``.
635 ValueError
636 Raised if the value is rejected by the ``check`` method.
637 """
638 if value is None: 638 ↛ 639line 638 didn't jump to line 639 because the condition on line 638 was never true
639 return
641 if not isinstance(value, self.dtype): 641 ↛ 642line 641 didn't jump to line 642 because the condition on line 641 was never true
642 msg = (
643 f"Value {value} is of incorrect type {_typeStr(value)}. Expected type {_typeStr(self.dtype)}"
644 )
645 raise TypeError(msg)
646 if self.check is not None and not self.check(value): 646 ↛ 647line 646 didn't jump to line 647 because the condition on line 646 was never true
647 msg = f"Value {value} is not a valid value"
648 raise ValueError(msg)
650 def _collectImports(self, instance, imports):
651 """Call the _collectImports method on all config
652 objects the field may own, and union them with the supplied imports
653 set.
655 Parameters
656 ----------
657 instance : instance or subclass of `lsst.pex.config.Config`
658 A config object that has this field defined on it
659 imports : `set`
660 Set of python modules that need imported after persistence
661 """
662 pass
664 def save(self, outfile, instance):
665 """Save this field to a file (for internal use only).
667 Parameters
668 ----------
669 outfile : file-like object
670 A writeable field handle.
671 instance : `~lsst.pex.config.Config`
672 The `~lsst.pex.config.Config` instance that contains this field.
674 Notes
675 -----
676 This method is invoked by the `~lsst.pex.config.Config` object that
677 contains this field and should not be called directly.
679 The output consists of the documentation string
680 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
681 line is formatted as an assignment: ``{fullname}={value}``.
683 This output can be executed with Python.
684 """
685 value = self.__get__(instance)
686 fullname = _joinNamePath(instance._name, self.name)
688 if self.deprecated and value == self.default: 688 ↛ 689line 688 didn't jump to line 689 because the condition on line 688 was never true
689 return
691 # write full documentation string as comment lines
692 # (i.e. first character is #)
693 doc = "# " + str(self.doc).replace("\n", "\n# ")
694 if isinstance(value, float) and not math.isfinite(value): 694 ↛ 696line 694 didn't jump to line 696 because the condition on line 694 was never true
695 # non-finite numbers need special care
696 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n")
697 else:
698 outfile.write(f"{doc}\n{fullname}={value!r}\n\n")
700 def toDict(self, instance):
701 """Convert the field value so that it can be set as the value of an
702 item in a `dict` (for internal use only).
704 Parameters
705 ----------
706 instance : `~lsst.pex.config.Config`
707 The `~lsst.pex.config.Config` that contains this field.
709 Returns
710 -------
711 value : object
712 The field's value. See *Notes*.
714 Notes
715 -----
716 This method invoked by the owning `~lsst.pex.config.Config` object and
717 should not be called directly.
719 Simple values are passed through. Complex data structures must be
720 manipulated. For example, a `~lsst.pex.config.Field` holding a
721 subconfig should, instead of the subconfig object, return a `dict`
722 where the keys are the field names in the subconfig, and the values are
723 the field values in the subconfig.
724 """
725 return self.__get__(instance)
727 def _copy_storage(self, old: Config, new: Config) -> Any:
728 """Copy the storage for this field in the given field into an object
729 suitable for storage in a new copy of that config.
731 Any frozen storage should be unfrozen.
732 """
733 return copy.deepcopy(old._storage[self.name])
735 @overload
736 def __get__(
737 self, instance: None, owner: Any = None, at: Any = None, label: str = "default"
738 ) -> Field[FieldTypeVar]: ...
740 @overload
741 def __get__(
742 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default"
743 ) -> FieldTypeVar: ...
745 def __get__(self, instance, owner=None, at=None, label="default"):
746 """Define how attribute access should occur on the Config instance
747 This is invoked by the owning config object and should not be called
748 directly.
750 When the field attribute is accessed on a Config class object, it
751 returns the field object itself in order to allow inspection of
752 Config classes.
754 When the field attribute is access on a config instance, the actual
755 value described by the field (and held by the Config instance) is
756 returned.
757 """
758 if instance is None: 758 ↛ 759line 758 didn't jump to line 759 because the condition on line 758 was never true
759 return self
760 else:
761 # try statements are almost free in python if they succeed
762 try:
763 return instance._storage[self.name]
764 except AttributeError:
765 if not isinstance(instance, Config):
766 return self
767 else:
768 raise AttributeError(
769 f"Config {instance} is missing _storage attribute, likely incorrectly initialized"
770 ) from None
772 def __set__(
773 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment"
774 ) -> None:
775 """Set an attribute on the config instance.
777 Parameters
778 ----------
779 instance : `lsst.pex.config.Config`
780 The config instance that contains this field.
781 value : obj
782 Value to set on this field.
783 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\
784 optional
785 The call stack (created by
786 `lsst.pex.config.callStack.getCallStack`).
787 label : `str`, optional
788 Event label for the history.
790 Notes
791 -----
792 This method is invoked by the owning `lsst.pex.config.Config` object
793 and should not be called directly.
795 Derived `~lsst.pex.config.Field` classes may need to override the
796 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
797 should follow the following rules:
799 - Do not allow modification of frozen configs.
800 - Validate the new value **before** modifying the field. Except if the
801 new value is `None`. `None` is special and no attempt should be made
802 to validate it until `lsst.pex.config.Config.validate` is called.
803 - Do not modify the `~lsst.pex.config.Config` instance to contain
804 invalid values.
805 - If the field is modified, update the history of the
806 `lsst.pex.config.field.Field` to reflect the changes.
808 In order to decrease the need to implement this method in derived
809 `~lsst.pex.config.Field` types, value validation is performed in the
810 `lsst.pex.config.Field._validateValue`. If only the validation step
811 differs in the derived `~lsst.pex.config.Field`, it is simpler to
812 implement `lsst.pex.config.Field._validateValue` than to reimplement
813 ``__set__``. More complicated behavior, however, may require
814 reimplementation.
815 """
816 if instance._frozen: 816 ↛ 817line 816 didn't jump to line 817 because the condition on line 816 was never true
817 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
819 history = instance._history.setdefault(self.name, [])
820 if value is not None: 820 ↛ 827line 820 didn't jump to line 827 because the condition on line 820 was always true
821 value = _autocast(value, self.dtype)
822 try:
823 self._validateValue(value)
824 except BaseException as e:
825 raise FieldValidationError(self, instance, str(e)) from e
827 instance._storage[self.name] = value
828 if at is None: 828 ↛ 829line 828 didn't jump to line 829 because the condition on line 828 was never true
829 at = getCallStack()
830 history.append((value, at, label))
832 def __delete__(self, instance, at=None, label="deletion"):
833 """Delete an attribute from a `lsst.pex.config.Config` instance.
835 Parameters
836 ----------
837 instance : `lsst.pex.config.Config`
838 The config instance that contains this field.
839 at : `list` of `lsst.pex.config.callStack.StackFrame`
840 The call stack (created by
841 `lsst.pex.config.callStack.getCallStack`).
842 label : `str`, optional
843 Event label for the history.
845 Notes
846 -----
847 This is invoked by the owning `~lsst.pex.config.Config` object and
848 should not be called directly.
849 """
850 if at is None:
851 at = getCallStack()
852 self.__set__(instance, None, at=at, label=label)
854 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
855 """Compare a field (named `Field.name`) in two
856 `~lsst.pex.config.Config` instances for equality.
858 Parameters
859 ----------
860 instance1 : `lsst.pex.config.Config`
861 Left-hand side `Config` instance to compare.
862 instance2 : `lsst.pex.config.Config`
863 Right-hand side `Config` instance to compare.
864 shortcut : `bool`, optional
865 **Unused.**
866 rtol : `float`, optional
867 Relative tolerance for floating point comparisons.
868 atol : `float`, optional
869 Absolute tolerance for floating point comparisons.
870 output : callable, optional
871 A callable that takes a string, used (possibly repeatedly) to
872 report inequalities.
874 Notes
875 -----
876 This method must be overridden by more complex `Field` subclasses.
878 See Also
879 --------
880 lsst.pex.config.compareScalars
881 """
882 v1 = getattr(instance1, self.name)
883 v2 = getattr(instance2, self.name)
884 name = getComparisonName(
885 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name)
886 )
887 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
890class RecordingImporter:
891 """Importer (for `sys.meta_path`) that records which modules are being
892 imported.
894 *This class does not do any importing itself.*
896 Examples
897 --------
898 Use this class as a context manager to ensure it is properly uninstalled
899 when done:
901 >>> with RecordingImporter() as importer:
902 ... # import stuff
903 ... import numpy as np
904 ... print("Imported: " + importer.getModules())
905 """
907 def __init__(self):
908 self._modules = set()
910 def __enter__(self):
911 self.origMetaPath = sys.meta_path
912 sys.meta_path = [self] + sys.meta_path # type: ignore
913 return self
915 def __exit__(self, *args):
916 self.uninstall()
917 return False # Don't suppress exceptions
919 def uninstall(self):
920 """Uninstall the importer."""
921 sys.meta_path = self.origMetaPath
923 def find_spec(self, fullname, path, target=None):
924 """Find a module.
926 Called as part of the ``import`` chain of events.
928 Parameters
929 ----------
930 fullname : `str`
931 Name of module.
932 path : `list` [`str`]
933 Search path. Unused.
934 target : `~typing.Any`, optional
935 Unused.
936 """
937 self._modules.add(fullname)
938 # Return None because we don't do any importing.
939 return None
941 def getModules(self):
942 """Get the set of modules that were imported.
944 Returns
945 -------
946 modules : `set` of `str`
947 Set of imported module names.
948 """
949 return self._modules
952# type ignore because type checker thinks ConfigMeta is Generic when it is not
953class Config(metaclass=ConfigMeta): # type: ignore
954 """Base class for configuration (*config*) objects.
956 Notes
957 -----
958 A ``Config`` object will usually have several `~lsst.pex.config.Field`
959 instances as class attributes. These are used to define most of the base
960 class behavior.
962 ``Config`` implements a mapping API that provides many `dict`-like methods,
963 such as `keys`, `values`, and `items`. ``Config`` instances also support
964 the ``in`` operator to test if a field is in the config. Unlike a `dict`,
965 ``Config`` classes are not subscriptable. Instead, access individual
966 fields as attributes of the configuration instance.
968 Examples
969 --------
970 Config classes are subclasses of ``Config`` that have
971 `~lsst.pex.config.Field` instances (or instances of
972 `~lsst.pex.config.Field` subclasses) as class attributes:
974 >>> from lsst.pex.config import Config, Field, ListField
975 >>> class DemoConfig(Config):
976 ... intField = Field(doc="An integer field", dtype=int, default=42)
977 ... listField = ListField(
978 ... doc="List of favorite beverages.",
979 ... dtype=str,
980 ... default=["coffee", "green tea", "water"],
981 ... )
982 >>> config = DemoConfig()
984 Configs support many `dict`-like APIs:
986 >>> config.keys()
987 ['intField', 'listField']
988 >>> "intField" in config
989 True
991 Individual fields can be accessed as attributes of the configuration:
993 >>> config.intField
994 42
995 >>> config.listField.append("earl grey tea")
996 >>> print(config.listField)
997 ['coffee', 'green tea', 'water', 'earl grey tea']
998 """
1000 _storage: dict[str, Any]
1001 _fields: dict[str, Field]
1002 _history: dict[str, list[Any]]
1003 _imports: set[Any]
1005 def __iter__(self):
1006 """Iterate over fields."""
1007 return self._fields.__iter__()
1009 def keys(self):
1010 """Get field names.
1012 Returns
1013 -------
1014 names : `~collections.abc.KeysView`
1015 List of `lsst.pex.config.Field` names.
1016 """
1017 return self._storage.keys()
1019 def values(self):
1020 """Get field values.
1022 Returns
1023 -------
1024 values : `~collections.abc.ValuesView`
1025 Iterator of field values.
1026 """
1027 return self._storage.values()
1029 def items(self):
1030 """Get configurations as ``(field name, field value)`` pairs.
1032 Returns
1033 -------
1034 items : `~collections.abc.ItemsView`
1035 Iterator of tuples for each configuration. Tuple items are:
1037 0. Field name.
1038 1. Field value.
1039 """
1040 return self._storage.items()
1042 def __contains__(self, name):
1043 """Return `True` if the specified field exists in this config.
1045 Parameters
1046 ----------
1047 name : `str`
1048 Field name to test for.
1050 Returns
1051 -------
1052 in : `bool`
1053 `True` if the specified field exists in the config.
1054 """
1055 return self._storage.__contains__(name)
1057 def __new__(cls, *args, **kw):
1058 """Allocate a new `lsst.pex.config.Config` object.
1060 In order to ensure that all Config object are always in a proper state
1061 when handed to users or to derived `~lsst.pex.config.Config` classes,
1062 some attributes are handled at allocation time rather than at
1063 initialization.
1065 This ensures that even if a derived `~lsst.pex.config.Config` class
1066 implements ``__init__``, its author does not need to be concerned about
1067 when or even the base ``Config.__init__`` should be called.
1068 """
1069 name = kw.pop("__name", None)
1070 at = kw.pop("__at", getCallStack())
1071 # remove __label and ignore it
1072 kw.pop("__label", "default")
1074 instance = object.__new__(cls)
1075 instance._frozen = False
1076 instance._name = name
1077 instance._storage = {}
1078 instance._history = {}
1079 instance._imports = set()
1080 # load up defaults
1081 for field in instance._fields.values():
1082 instance._history[field.name] = []
1083 field.__set__(instance, field.default, at=at + [field.source], label="default")
1084 # set custom default-overrides
1085 instance.setDefaults()
1086 # set constructor overrides
1087 instance.update(__at=at, **kw)
1088 return instance
1090 def copy(self) -> Config:
1091 """Return a deep copy of this config.
1093 Notes
1094 -----
1095 The returned config object is not frozen, even if the original was.
1096 If a nested config object is copied, it retains the name from its
1097 original hierarchy.
1099 Nested objects are only shared between the new and old configs if they
1100 are not possible to modify via the config's interfaces (e.g. entries
1101 in the the history list are not copied, but the lists themselves are,
1102 so modifications to one copy do not modify the other).
1103 """
1104 instance = object.__new__(type(self))
1105 instance._frozen = False
1106 instance._name = self._name
1107 instance._history = {k: list(v) for k, v in self._history.items()}
1108 instance._imports = set(self._imports)
1109 # Important to set up storage last, since fields sometimes store
1110 # proxy objects that reference their parent (especially for history).
1111 instance._storage = {k: self._fields[k]._copy_storage(self, instance) for k in self._storage}
1112 return instance
1114 def __reduce__(self):
1115 """Reduction for pickling (function with arguments to reproduce).
1117 We need to condense and reconstitute the `~lsst.pex.config.Config`,
1118 since it may contain lambdas (as the ``check`` elements) that cannot
1119 be pickled.
1120 """
1121 # The stream must be in characters to match the API but pickle
1122 # requires bytes
1123 stream = io.StringIO()
1124 self.saveToStream(stream)
1125 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
1127 def setDefaults(self):
1128 """Subclass hook for computing defaults.
1130 Notes
1131 -----
1132 Derived `~lsst.pex.config.Config` classes that must compute defaults
1133 rather than using the `~lsst.pex.config.Field` instances's defaults
1134 should do so here. To correctly use inherited defaults,
1135 implementations of ``setDefaults`` must call their base class's
1136 ``setDefaults``.
1137 """
1138 pass
1140 def update(self, **kw):
1141 """Update values of fields specified by the keyword arguments.
1143 Parameters
1144 ----------
1145 **kw
1146 Keywords are configuration field names. Values are configuration
1147 field values.
1149 Notes
1150 -----
1151 The ``__at`` and ``__label`` keyword arguments are special internal
1152 keywords. They are used to strip out any internal steps from the
1153 history tracebacks of the config. Do not modify these keywords to
1154 subvert a `~lsst.pex.config.Config` instance's history.
1156 Examples
1157 --------
1158 This is a config with three fields:
1160 >>> from lsst.pex.config import Config, Field
1161 >>> class DemoConfig(Config):
1162 ... fieldA = Field(doc="Field A", dtype=int, default=42)
1163 ... fieldB = Field(doc="Field B", dtype=bool, default=True)
1164 ... fieldC = Field(doc="Field C", dtype=str, default="Hello world")
1165 >>> config = DemoConfig()
1167 These are the default values of each field:
1169 >>> for name, value in config.iteritems():
1170 ... print(f"{name}: {value}")
1171 fieldA: 42
1172 fieldB: True
1173 fieldC: 'Hello world'
1175 Using this method to update ``fieldA`` and ``fieldC``:
1177 >>> config.update(fieldA=13, fieldC="Updated!")
1179 Now the values of each field are:
1181 >>> for name, value in config.iteritems():
1182 ... print(f"{name}: {value}")
1183 fieldA: 13
1184 fieldB: True
1185 fieldC: 'Updated!'
1186 """
1187 at = kw.pop("__at", getCallStack())
1188 label = kw.pop("__label", "update")
1190 for name, value in kw.items():
1191 try:
1192 field = self._fields[name]
1193 field.__set__(self, value, at=at, label=label)
1194 except KeyError as e:
1195 e.add_note(f"No field of name {name} exists in config type {_typeStr(self)}")
1196 raise
1198 def _filename_to_resource(
1199 self, filename: ResourcePathExpression | None = None
1200 ) -> tuple[ResourcePath | None, str]:
1201 """Create resource path from filename.
1203 Parameters
1204 ----------
1205 filename : `lsst.resources.ResourcePathExpression` or `None`
1206 The URI expression associated with this config. Can be `None`
1207 if no file URI is known.
1209 Returns
1210 -------
1211 resource : `lsst.resources.ResourcePath` or `None`
1212 The resource version of the filename. Returns `None` if no filename
1213 was given or refers to unspecified value.
1214 file_string : `str`
1215 String form of the resource for use in ``__file__``
1216 """
1217 if filename is None or filename in ("?", "<unknown>"): 1217 ↛ 1219line 1217 didn't jump to line 1219 because the condition on line 1217 was always true
1218 return None, "<unknown>"
1219 base = _get_config_root()
1220 resource = ResourcePath(filename, forceAbsolute=True, forceDirectory=False, root=base)
1222 # Preferred definition of __file__ is the full OS path. If a config
1223 # is loaded with a relative path it must be converted to the absolute
1224 # path to avoid confusion with later relative paths referenced inside
1225 # the config.
1226 if resource.scheme == "file":
1227 file_string = resource.ospath
1228 else:
1229 file_string = str(resource)
1231 return resource, file_string
1233 def load(self, filename, root="config"):
1234 """Modify this config in place by executing the Python code in a
1235 configuration file.
1237 Parameters
1238 ----------
1239 filename : `lsst.resources.ResourcePathExpression`
1240 Name of the configuration URI. A configuration file is a Python
1241 module. Since configuration files are Python code, remote URIs
1242 are not allowed.
1243 root : `str`, optional
1244 Name of the variable in file that refers to the config being
1245 overridden.
1247 For example, the value of root is ``"config"`` and the file
1248 contains::
1250 config.myField = 5
1252 Then this config's field ``myField`` is set to ``5``.
1254 See Also
1255 --------
1256 lsst.pex.config.Config.loadFromStream
1257 lsst.pex.config.Config.loadFromString
1258 lsst.pex.config.Config.save
1259 lsst.pex.config.Config.saveToStream
1260 lsst.pex.config.Config.saveToString
1261 """
1262 resource, file_string = self._filename_to_resource(filename)
1263 if resource is None:
1264 # A filename is required.
1265 raise ValueError(f"Undefined URI provided to load command: {filename}.")
1267 if resource.scheme not in ("file", "eups", "resource"):
1268 raise ValueError(f"Remote URI ({resource}) can not be used to load configurations.")
1270 # Push the directory of the file we are now reading onto the stack
1271 # so that nested loads are relative to this file.
1272 with _push_config_root(resource.dirname()):
1273 _LOG.debug("Updating config from URI %s", str(resource))
1274 with resource.open("r") as f:
1275 code = compile(f.read(), filename=file_string, mode="exec")
1276 self._loadFromString(code, root=root, filename=file_string)
1278 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None):
1279 """Modify this Config in place by executing the Python code in the
1280 provided stream.
1282 Parameters
1283 ----------
1284 stream : file-like object, `str`, `bytes`, or `~types.CodeType`
1285 Stream containing configuration override code. If this is a
1286 code object, it should be compiled with ``mode="exec"``.
1287 root : `str`, optional
1288 Name of the variable in file that refers to the config being
1289 overridden.
1291 For example, the value of root is ``"config"`` and the file
1292 contains::
1294 config.myField = 5
1296 Then this config's field ``myField`` is set to ``5``.
1297 filename : `str`, optional
1298 Name of the configuration file, or `None` if unknown or contained
1299 in the stream. Used for error reporting and to set ``__file__``
1300 variable in config.
1301 extraLocals : `dict` of `str` to `object`, optional
1302 Any extra variables to include in local scope when loading.
1304 Notes
1305 -----
1306 For backwards compatibility reasons, this method accepts strings, bytes
1307 and code objects as well as file-like objects. New code should use
1308 `loadFromString` instead for most of these types.
1310 See Also
1311 --------
1312 lsst.pex.config.Config.load
1313 lsst.pex.config.Config.loadFromString
1314 lsst.pex.config.Config.save
1315 lsst.pex.config.Config.saveToStream
1316 lsst.pex.config.Config.saveToString
1317 """
1318 if hasattr(stream, "read"): 1318 ↛ 1319line 1318 didn't jump to line 1319 because the condition on line 1318 was never true
1319 if filename is None:
1320 filename = getattr(stream, "name", "<unknown>")
1321 code = compile(stream.read(), filename=filename, mode="exec")
1322 else:
1323 code = stream
1324 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1326 def loadFromString(self, code, root="config", filename=None, extraLocals=None):
1327 """Modify this Config in place by executing the Python code in the
1328 provided string.
1330 Parameters
1331 ----------
1332 code : `str`, `bytes`, or `~types.CodeType`
1333 Stream containing configuration override code.
1334 root : `str`, optional
1335 Name of the variable in file that refers to the config being
1336 overridden.
1338 For example, the value of root is ``"config"`` and the file
1339 contains::
1341 config.myField = 5
1343 Then this config's field ``myField`` is set to ``5``.
1344 filename : `lsst.resources.ResourcePathExpression`, optional
1345 URI of the configuration file, or `None` if unknown or contained
1346 in the stream. Used for error reporting and to set ``__file__``
1347 variable. Required to be set if the string config attempts to
1348 load other configs using either relative path or ``__file__``.
1349 extraLocals : `dict` of `str` to `object`, optional
1350 Any extra variables to include in local scope when loading.
1352 Raises
1353 ------
1354 ValueError
1355 Raised if a key in extraLocals is the same value as the value of
1356 the root argument.
1358 See Also
1359 --------
1360 lsst.pex.config.Config.load
1361 lsst.pex.config.Config.loadFromStream
1362 lsst.pex.config.Config.save
1363 lsst.pex.config.Config.saveToStream
1364 lsst.pex.config.Config.saveToString
1365 """
1366 if filename is None: 1366 ↛ 1371line 1366 didn't jump to line 1371 because the condition on line 1366 was always true
1367 # try to determine the file name; a compiled string
1368 # has attribute "co_filename",
1369 filename = getattr(code, "co_filename", "<unknown>")
1371 resource, file_string = self._filename_to_resource(filename)
1372 if resource is None: 1372 ↛ 1379line 1372 didn't jump to line 1379 because the condition on line 1372 was always true
1373 # No idea where this config came from so no ability to deal with
1374 # relative paths. No reason to use context.
1375 self._loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1376 else:
1377 # Push the directory of the file we are now reading onto the stack
1378 # so that nested loads are relative to this file.
1379 with _push_config_root(resource.dirname()):
1380 self._loadFromString(code, root=root, filename=file_string, extraLocals=extraLocals)
1382 def _loadFromString(self, code, root="config", filename=None, extraLocals=None):
1383 """Update config from string.
1385 Assumes relative directory path context has been setup by caller.
1386 """
1387 with RecordingImporter() as importer:
1388 globals = {"__file__": filename}
1389 local = {root: self}
1390 if extraLocals is not None: 1390 ↛ 1392line 1390 didn't jump to line 1392 because the condition on line 1390 was never true
1391 # verify the value of root was not passed as extra local args
1392 if root in extraLocals:
1393 raise ValueError(
1394 f"{root} is reserved and cannot be used as a variable name in extraLocals"
1395 )
1396 local.update(extraLocals)
1397 exec(code, globals, local)
1399 self._imports.update(importer.getModules())
1401 def save(self, filename, root="config"):
1402 """Save a Python script to the named file, which, when loaded,
1403 reproduces this config.
1405 Parameters
1406 ----------
1407 filename : `str`
1408 Desination filename of this configuration.
1409 root : `str`, optional
1410 Name to use for the root config variable. The same value must be
1411 used when loading (see `lsst.pex.config.Config.load`).
1413 See Also
1414 --------
1415 lsst.pex.config.Config.saveToStream
1416 lsst.pex.config.Config.saveToString
1417 lsst.pex.config.Config.load
1418 lsst.pex.config.Config.loadFromStream
1419 lsst.pex.config.Config.loadFromString
1420 """
1421 d = os.path.dirname(filename)
1422 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1423 self.saveToStream(outfile, root)
1424 # tempfile is hardcoded to create files with mode '0600'
1425 # for an explantion of these antics see:
1426 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1427 umask = os.umask(0o077)
1428 os.umask(umask)
1429 os.chmod(outfile.name, (~umask & 0o666))
1430 # chmod before the move so we get quasi-atomic behavior if the
1431 # source and dest. are on the same filesystem.
1432 # os.rename may not work across filesystems
1433 shutil.move(outfile.name, filename)
1435 def saveToString(self, skipImports=False):
1436 """Return the Python script form of this configuration as an executable
1437 string.
1439 Parameters
1440 ----------
1441 skipImports : `bool`, optional
1442 If `True` then do not include ``import`` statements in output,
1443 this is to support human-oriented output from ``pipetask`` where
1444 additional clutter is not useful.
1446 Returns
1447 -------
1448 code : `str`
1449 A code string readable by `loadFromString`.
1451 See Also
1452 --------
1453 lsst.pex.config.Config.save
1454 lsst.pex.config.Config.saveToStream
1455 lsst.pex.config.Config.load
1456 lsst.pex.config.Config.loadFromStream
1457 lsst.pex.config.Config.loadFromString
1458 """
1459 buffer = io.StringIO()
1460 self.saveToStream(buffer, skipImports=skipImports)
1461 return buffer.getvalue()
1463 def saveToStream(self, outfile, root="config", skipImports=False):
1464 """Save a configuration file to a stream, which, when loaded,
1465 reproduces this config.
1467 Parameters
1468 ----------
1469 outfile : file-like object
1470 Destination file object write the config into. Accepts strings not
1471 bytes.
1472 root : `str`, optional
1473 Name to use for the root config variable. The same value must be
1474 used when loading (see `lsst.pex.config.Config.load`).
1475 skipImports : `bool`, optional
1476 If `True` then do not include ``import`` statements in output,
1477 this is to support human-oriented output from ``pipetask`` where
1478 additional clutter is not useful.
1480 See Also
1481 --------
1482 lsst.pex.config.Config.save
1483 lsst.pex.config.Config.saveToString
1484 lsst.pex.config.Config.load
1485 lsst.pex.config.Config.loadFromStream
1486 lsst.pex.config.Config.loadFromString
1487 """
1488 tmp = self._name
1489 self._rename(root)
1490 try:
1491 if not skipImports: 1491 ↛ 1508line 1491 didn't jump to line 1508 because the condition on line 1491 was always true
1492 self._collectImports()
1493 # Remove self from the set, as it is handled explicitly below
1494 self._imports.remove(self.__module__)
1495 configType = type(self)
1496 typeString = _typeStr(configType)
1497 outfile.write(f"import {configType.__module__}\n")
1498 # We are required to write this on a single line because
1499 # of later regex matching, rather than adopting black style
1500 # formatting.
1501 outfile.write(
1502 f'assert type({root}) is {typeString}, f"config is of type '
1503 f'{{type({root}).__module__}}.{{type({root}).__name__}} instead of {typeString}"\n\n'
1504 )
1505 for imp in sorted(self._imports): 1505 ↛ 1506line 1505 didn't jump to line 1506 because the loop on line 1505 never started
1506 if imp in sys.modules and sys.modules[imp] is not None:
1507 outfile.write(f"import {imp}\n")
1508 self._save(outfile)
1509 finally:
1510 self._rename(tmp)
1512 def freeze(self):
1513 """Make this config, and all subconfigs, read-only."""
1514 self._frozen = True
1515 for field in self._fields.values():
1516 field.freeze(self)
1518 def _save(self, outfile):
1519 """Save this config to an open stream object.
1521 Parameters
1522 ----------
1523 outfile : file-like object
1524 Destination file object write the config into. Accepts strings not
1525 bytes.
1526 """
1527 for field in self._fields.values():
1528 field.save(outfile, self)
1530 def _collectImports(self):
1531 """Add module containing self to the list of things to import and
1532 then loops over all the fields in the config calling a corresponding
1533 collect method.
1535 The field method will call _collectImports on any
1536 configs it may own and return the set of things to import. This
1537 returned set will be merged with the set of imports for this config
1538 class.
1539 """
1540 self._imports.add(self.__module__)
1541 for field in self._fields.values():
1542 field._collectImports(self, self._imports)
1544 def toDict(self):
1545 """Make a dictionary of field names and their values.
1547 Returns
1548 -------
1549 dict_ : `dict`
1550 Dictionary with keys that are `~lsst.pex.config.Field` names.
1551 Values are `~lsst.pex.config.Field` values.
1553 See Also
1554 --------
1555 lsst.pex.config.Field.toDict
1557 Notes
1558 -----
1559 This method uses the `~lsst.pex.config.Field.toDict` method of
1560 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1561 implement a ``toDict`` method for *this* method to work.
1562 """
1563 dict_ = {}
1564 for name, field in self._fields.items():
1565 dict_[name] = field.toDict(self)
1566 return dict_
1568 def names(self):
1569 """Get all the field names in the config, recursively.
1571 Returns
1572 -------
1573 names : `list` of `str`
1574 Field names.
1575 """
1576 #
1577 # Rather than sort out the recursion all over again use the
1578 # pre-existing saveToStream()
1579 #
1580 with io.StringIO() as strFd:
1581 self.saveToStream(strFd, "config")
1582 contents = strFd.getvalue()
1583 strFd.close()
1584 #
1585 # Pull the names out of the dumped config
1586 #
1587 keys = []
1588 for line in contents.split("\n"):
1589 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1590 continue
1592 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1593 if mat:
1594 keys.append(mat.group(1))
1596 return keys
1598 def _rename(self, name):
1599 """Rename this config object in its parent `~lsst.pex.config.Config`.
1601 Parameters
1602 ----------
1603 name : `str`
1604 New name for this config in its parent `~lsst.pex.config.Config`.
1606 Notes
1607 -----
1608 This method uses the `~lsst.pex.config.Field.rename` method of
1609 individual `lsst.pex.config.Field` instances.
1610 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1611 method for *this* method to work.
1613 See Also
1614 --------
1615 lsst.pex.config.Field.rename
1616 """
1617 self._name = name
1618 for field in self._fields.values():
1619 field.rename(self)
1621 def validate(self):
1622 """Validate the Config, raising an exception if invalid.
1624 Raises
1625 ------
1626 lsst.pex.config.FieldValidationError
1627 Raised if verification fails.
1629 Notes
1630 -----
1631 The base class implementation performs type checks on all fields by
1632 calling their `~lsst.pex.config.Field.validate` methods.
1634 Complex single-field validation can be defined by deriving new Field
1635 types. For convenience, some derived `lsst.pex.config.Field`-types
1636 (`~lsst.pex.config.ConfigField` and
1637 `~lsst.pex.config.ConfigChoiceField`) are defined in
1638 ``lsst.pex.config`` that handle recursing into subconfigs.
1640 Inter-field relationships should only be checked in derived
1641 `~lsst.pex.config.Config` classes after calling this method, and base
1642 validation is complete.
1643 """
1644 for field in self._fields.values():
1645 field.validate(self)
1647 def formatHistory(self, name, **kwargs):
1648 """Format a configuration field's history to a human-readable format.
1650 Parameters
1651 ----------
1652 name : `str`
1653 Name of a `~lsst.pex.config.Field` in this config.
1654 **kwargs
1655 Keyword arguments passed to `lsst.pex.config.history.format`.
1657 Returns
1658 -------
1659 history : `str`
1660 A string containing the formatted history.
1662 See Also
1663 --------
1664 lsst.pex.config.history.format
1665 """
1666 import lsst.pex.config.history as pexHist
1668 return pexHist.format(self, name, **kwargs)
1670 history = property(lambda x: x._history)
1671 """Read-only history.
1672 """
1674 def __setattr__(self, attr, value, at=None, label="assignment"):
1675 """Set an attribute (such as a field's value).
1677 Notes
1678 -----
1679 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1680 locked such that no additional attributes nor properties may be added
1681 to them dynamically.
1683 Although this is not the standard Python behavior, it helps to protect
1684 users from accidentally mispelling a field name, or trying to set a
1685 non-existent field.
1686 """
1687 if attr in self._fields:
1688 if self._fields[attr].deprecated is not None: 1688 ↛ 1689line 1688 didn't jump to line 1689 because the condition on line 1688 was never true
1689 fullname = _joinNamePath(self._name, self._fields[attr].name)
1690 warnings.warn(
1691 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1692 FutureWarning,
1693 stacklevel=2,
1694 )
1695 if at is None: 1695 ↛ 1698line 1695 didn't jump to line 1698 because the condition on line 1695 was always true
1696 at = getCallStack()
1697 # This allows Field descriptors to work.
1698 self._fields[attr].__set__(self, value, at=at, label=label)
1699 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1699 ↛ 1701line 1699 didn't jump to line 1701 because the condition on line 1699 was never true
1700 # This allows properties and other non-Field descriptors to work.
1701 return object.__setattr__(self, attr, value)
1702 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1702 ↛ 1707line 1702 didn't jump to line 1707 because the condition on line 1702 was always true
1703 # This allows specific private attributes to work.
1704 self.__dict__[attr] = value
1705 else:
1706 # We throw everything else.
1707 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
1709 def __delattr__(self, attr, at=None, label="deletion"):
1710 if attr in self._fields:
1711 if at is None:
1712 at = getCallStack()
1713 self._fields[attr].__delete__(self, at=at, label=label)
1714 else:
1715 object.__delattr__(self, attr)
1717 def __eq__(self, other):
1718 if type(other) is type(self): 1718 ↛ 1719line 1718 didn't jump to line 1719 because the condition on line 1718 was never true
1719 for name in self._fields:
1720 thisValue = getattr(self, name)
1721 otherValue = getattr(other, name)
1722 if isinstance(thisValue, float) and math.isnan(thisValue):
1723 if not math.isnan(otherValue):
1724 return False
1725 elif thisValue != otherValue:
1726 return False
1727 return True
1728 return False
1730 def __ne__(self, other):
1731 return not self.__eq__(other)
1733 def __str__(self):
1734 return str(self.toDict())
1736 def __repr__(self):
1737 return "{}({})".format(
1738 _typeStr(self),
1739 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None),
1740 )
1742 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None):
1743 """Compare this configuration to another `~lsst.pex.config.Config` for
1744 equality.
1746 Parameters
1747 ----------
1748 other : `lsst.pex.config.Config`
1749 Other `~lsst.pex.config.Config` object to compare against this
1750 config.
1751 shortcut : `bool`, optional
1752 If `True`, return as soon as an inequality is found. Default is
1753 `True`.
1754 rtol : `float`, optional
1755 Relative tolerance for floating point comparisons.
1756 atol : `float`, optional
1757 Absolute tolerance for floating point comparisons.
1758 output : callable, optional
1759 A callable that takes a string, used (possibly repeatedly) to
1760 report inequalities.
1762 Returns
1763 -------
1764 isEqual : `bool`
1765 `True` when the two `lsst.pex.config.Config` instances are equal.
1766 `False` if there is an inequality.
1768 See Also
1769 --------
1770 lsst.pex.config.compareConfigs
1772 Notes
1773 -----
1774 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1775 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1776 are not considered by this method.
1778 Floating point comparisons are performed by `numpy.allclose`.
1779 """
1780 name1 = self._name if self._name is not None else "config"
1781 name2 = other._name if other._name is not None else "config"
1782 name = getComparisonName(name1, name2)
1783 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output)
1785 @classmethod
1786 def __init_subclass__(cls, **kwargs):
1787 """Run initialization for every subclass.
1789 Specifically registers the subclass with a YAML representer
1790 and YAML constructor (if pyyaml is available)
1791 """
1792 super().__init_subclass__(**kwargs)
1794 if not yaml: 1794 ↛ 1795line 1794 didn't jump to line 1795 because the condition on line 1794 was never true
1795 return
1797 yaml.add_representer(cls, _yaml_config_representer)
1799 @classmethod
1800 def _fromPython(cls, config_py):
1801 """Instantiate a `Config`-subclass from serialized Python form.
1803 Parameters
1804 ----------
1805 config_py : `str`
1806 A serialized form of the Config as created by
1807 `Config.saveToStream`.
1809 Returns
1810 -------
1811 config : `Config`
1812 Reconstructed `Config` instant.
1813 """
1814 cls = _classFromPython(config_py)
1815 return unreduceConfig(cls, config_py)
1818def _classFromPython(config_py):
1819 """Return the Config subclass required by this Config serialization.
1821 Parameters
1822 ----------
1823 config_py : `str`
1824 A serialized form of the Config as created by
1825 `Config.saveToStream`.
1827 Returns
1828 -------
1829 cls : `type`
1830 The `Config` subclass associated with this config.
1831 """
1832 # standard serialization has the form:
1833 # import config.class
1834 # assert type(config) is config.class.Config, ...
1835 # Older files use "type(config)==" instead.
1836 # We want to parse these two lines so we can get the class itself
1838 # Do a single regex to avoid large string copies when splitting a
1839 # large config into separate lines.
1840 # The assert regex cannot be greedy because the assert error string
1841 # can include both "," and " is ".
1842 matches = re.search(r"^import ([\w.]+)\nassert type\(\S+\)(?:\s*==\s*| is )(.*?),", config_py)
1844 if not matches:
1845 first_line, second_line, _ = config_py.split("\n", 2)
1846 raise ValueError(
1847 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}"
1848 )
1850 module_name = matches.group(1)
1851 module = importlib.import_module(module_name)
1853 # Second line
1854 full_name = matches.group(2)
1856 # Remove the module name from the full name
1857 if not full_name.startswith(module_name):
1858 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1860 # if module name is a.b.c and full name is a.b.c.d.E then
1861 # we need to remove a.b.c. and iterate over the remainder
1862 # The +1 is for the extra dot after a.b.c
1863 remainder = full_name[len(module_name) + 1 :]
1864 components = remainder.split(".")
1865 pytype = module
1866 for component in components:
1867 pytype = getattr(pytype, component)
1868 return pytype
1871def unreduceConfig(cls_, stream):
1872 """Create a `~lsst.pex.config.Config` from a stream.
1874 Parameters
1875 ----------
1876 cls_ : `lsst.pex.config.Config`-type
1877 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1878 with configurations in the ``stream``.
1879 stream : file-like object, `str`, or `~types.CodeType`
1880 Stream containing configuration override code.
1882 Returns
1883 -------
1884 config : `lsst.pex.config.Config`
1885 Config instance.
1887 See Also
1888 --------
1889 lsst.pex.config.Config.loadFromStream
1890 """
1891 config = cls_()
1892 config.loadFromStream(stream)
1893 return config