Coverage for python/lsst/pex/config/config.py : 63%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28__all__ = ("Config", "ConfigMeta", "Field", "FieldValidationError")
30import io
31import os
32import re
33import sys
34import math
35import copy
36import tempfile
37import shutil
38import warnings
40from .comparison import getComparisonName, compareScalars, compareConfigs
41from .callStack import getStackFrame, getCallStack
44def _joinNamePath(prefix=None, name=None, index=None):
45 """Generate nested configuration names.
46 """
47 if not prefix and not name: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true
48 raise ValueError("Invalid name: cannot be None")
49 elif not name: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true
50 name = prefix
51 elif prefix and name: 51 ↛ 54line 51 didn't jump to line 54, because the condition on line 51 was never false
52 name = prefix + "." + name
54 if index is not None: 54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true
55 return "%s[%r]" % (name, index)
56 else:
57 return name
60def _autocast(x, dtype):
61 """Cast a value to a type, if appropriate.
63 Parameters
64 ----------
65 x : object
66 A value.
67 dtype : tpye
68 Data type, such as `float`, `int`, or `str`.
70 Returns
71 -------
72 values : object
73 If appropriate, the returned value is ``x`` cast to the given type
74 ``dtype``. If the cast cannot be performed the original value of
75 ``x`` is returned.
76 """
77 if dtype == float and isinstance(x, int):
78 return float(x)
79 return x
82def _typeStr(x):
83 """Generate a fully-qualified type name.
85 Returns
86 -------
87 `str`
88 Fully-qualified type name.
90 Notes
91 -----
92 This function is used primarily for writing config files to be executed
93 later upon with the 'load' function.
94 """
95 if hasattr(x, '__module__') and hasattr(x, '__name__'):
96 xtype = x
97 else:
98 xtype = type(x)
99 if (sys.version_info.major <= 2 and xtype.__module__ == '__builtin__') or xtype.__module__ == 'builtins': 99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true
100 return xtype.__name__
101 else:
102 return "%s.%s" % (xtype.__module__, xtype.__name__)
105class ConfigMeta(type):
106 """A metaclass for `lsst.pex.config.Config`.
108 Notes
109 -----
110 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
111 class attributes as a class attribute called ``_fields``, and adds
112 the name of each field as an instance variable of the field itself (so you
113 don't have to pass the name of the field to the field constructor).
114 """
116 def __init__(cls, name, bases, dict_):
117 type.__init__(cls, name, bases, dict_)
118 cls._fields = {}
119 cls._source = getStackFrame()
121 def getFields(classtype):
122 fields = {}
123 bases = list(classtype.__bases__)
124 bases.reverse()
125 for b in bases:
126 fields.update(getFields(b))
128 for k, v in classtype.__dict__.items():
129 if isinstance(v, Field):
130 fields[k] = v
131 return fields
133 fields = getFields(cls)
134 for k, v in fields.items():
135 setattr(cls, k, copy.deepcopy(v))
137 def __setattr__(cls, name, value):
138 if isinstance(value, Field):
139 value.name = name
140 cls._fields[name] = value
141 type.__setattr__(cls, name, value)
144class FieldValidationError(ValueError):
145 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
146 particular ``~lsst.pex.config.Config``.
148 Parameters
149 ----------
150 field : `lsst.pex.config.Field`
151 The field that was not valid.
152 config : `lsst.pex.config.Config`
153 The config containing the invalid field.
154 msg : `str`
155 Text describing why the field was not valid.
156 """
158 def __init__(self, field, config, msg):
159 self.fieldType = type(field)
160 """Type of the `~lsst.pex.config.Field` that incurred the error.
161 """
163 self.fieldName = field.name
164 """Name of the `~lsst.pex.config.Field` instance that incurred the
165 error (`str`).
167 See also
168 --------
169 lsst.pex.config.Field.name
170 """
172 self.fullname = _joinNamePath(config._name, field.name)
173 """Fully-qualified name of the `~lsst.pex.config.Field` instance
174 (`str`).
175 """
177 self.history = config.history.setdefault(field.name, [])
178 """Full history of all changes to the `~lsst.pex.config.Field`
179 instance.
180 """
182 self.fieldSource = field.source
183 """File and line number of the `~lsst.pex.config.Field` definition.
184 """
186 self.configSource = config._source
187 error = "%s '%s' failed validation: %s\n"\
188 "For more information see the Field definition at:\n%s"\
189 " and the Config definition at:\n%s" % \
190 (self.fieldType.__name__, self.fullname, msg,
191 self.fieldSource.format(), self.configSource.format())
192 super().__init__(error)
195class Field:
196 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
197 `complex`, `bool`, and `str` data types.
199 Parameters
200 ----------
201 doc : `str`
202 A description of the field for users.
203 dtype : type
204 The field's data type. ``Field`` only supports basic data types:
205 `int`, `float`, `complex`, `bool`, and `str`. See
206 `Field.supportedTypes`.
207 default : object, optional
208 The field's default value.
209 check : callable, optional
210 A callable that is called with the field's value. This callable should
211 return `False` if the value is invalid. More complex inter-field
212 validation can be written as part of the
213 `lsst.pex.config.Config.validate` method.
214 optional : `bool`, optional
215 This sets whether the field is considered optional, and therefore
216 doesn't need to be set by the user. When `False`,
217 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
218 deprecated : None or `str`, optional
219 A description of why this Field is deprecated, including removal date.
220 If not None, the string is appended to the docstring for this Field.
222 Raises
223 ------
224 ValueError
225 Raised when the ``dtype`` parameter is not one of the supported types
226 (see `Field.supportedTypes`).
228 See also
229 --------
230 ChoiceField
231 ConfigChoiceField
232 ConfigDictField
233 ConfigField
234 ConfigurableField
235 DictField
236 ListField
237 RangeField
238 RegistryField
240 Notes
241 -----
242 ``Field`` instances (including those of any subclass of ``Field``) are used
243 as class attributes of `~lsst.pex.config.Config` subclasses (see the
244 example, below). ``Field`` attributes work like the `property` attributes
245 of classes that implement custom setters and getters. `Field` attributes
246 belong to the class, but operate on the instance. Formally speaking,
247 `Field` attributes are `descriptors
248 <https://docs.python.org/3/howto/descriptor.html>`_.
250 When you access a `Field` attribute on a `Config` instance, you don't
251 get the `Field` instance itself. Instead, you get the value of that field,
252 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
253 container type (like a `lsst.pex.config.List`) depending on the field's
254 type. See the example, below.
256 Examples
257 --------
258 Instances of ``Field`` should be used as class attributes of
259 `lsst.pex.config.Config` subclasses:
261 >>> from lsst.pex.config import Config, Field
262 >>> class Example(Config):
263 ... myInt = Field("An integer field.", int, default=0)
264 ...
265 >>> print(config.myInt)
266 0
267 >>> config.myInt = 5
268 >>> print(config.myInt)
269 5
270 """
272 supportedTypes = set((str, bool, float, int, complex))
273 """Supported data types for field values (`set` of types).
274 """
276 def __init__(self, doc, dtype, default=None, check=None, optional=False, deprecated=None):
277 if dtype not in self.supportedTypes: 277 ↛ 278line 277 didn't jump to line 278, because the condition on line 277 was never true
278 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
280 source = getStackFrame()
281 self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source,
282 deprecated=deprecated)
284 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
285 """Set attributes, usually during initialization.
286 """
287 self.dtype = dtype
288 """Data type for the field.
289 """
291 # append the deprecation message to the docstring.
292 if deprecated is not None:
293 doc = f"{doc} Deprecated: {deprecated}"
294 self.doc = doc
295 """A description of the field (`str`).
296 """
298 self.deprecated = deprecated
299 """If not None, a description of why this field is deprecated (`str`).
300 """
302 self.__doc__ = f"{doc} (`{dtype.__name__}`"
303 if optional or default is not None:
304 self.__doc__ += f", default ``{default!r}``"
305 self.__doc__ += ")"
307 self.default = default
308 """Default value for this field.
309 """
311 self.check = check
312 """A user-defined function that validates the value of the field.
313 """
315 self.optional = optional
316 """Flag that determines if the field is required to be set (`bool`).
318 When `False`, `lsst.pex.config.Config.validate` will fail if the
319 field's value is `None`.
320 """
322 self.source = source
323 """The stack frame where this field is defined (`list` of
324 `lsst.pex.config.callStack.StackFrame`).
325 """
327 def rename(self, instance):
328 """Rename the field in a `~lsst.pex.config.Config` (for internal use
329 only).
331 Parameters
332 ----------
333 instance : `lsst.pex.config.Config`
334 The config instance that contains this field.
336 Notes
337 -----
338 This method is invoked by the `lsst.pex.config.Config` object that
339 contains this field and should not be called directly.
341 Renaming is only relevant for `~lsst.pex.config.Field` instances that
342 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should
343 rename each subconfig with the full field name as generated by
344 `lsst.pex.config.config._joinNamePath`.
345 """
346 pass
348 def validate(self, instance):
349 """Validate the field (for internal use only).
351 Parameters
352 ----------
353 instance : `lsst.pex.config.Config`
354 The config instance that contains this field.
356 Raises
357 ------
358 lsst.pex.config.FieldValidationError
359 Raised if verification fails.
361 Notes
362 -----
363 This method provides basic validation:
365 - Ensures that the value is not `None` if the field is not optional.
366 - Ensures type correctness.
367 - Ensures that the user-provided ``check`` function is valid.
369 Most `~lsst.pex.config.Field` subclasses should call
370 `lsst.pex.config.field.Field.validate` if they re-implement
371 `~lsst.pex.config.field.Field.validate`.
372 """
373 value = self.__get__(instance)
374 if not self.optional and value is None:
375 raise FieldValidationError(self, instance, "Required value cannot be None")
377 def freeze(self, instance):
378 """Make this field read-only (for internal use only).
380 Parameters
381 ----------
382 instance : `lsst.pex.config.Config`
383 The config instance that contains this field.
385 Notes
386 -----
387 Freezing is only relevant for fields that hold subconfigs. Fields which
388 hold subconfigs should freeze each subconfig.
390 **Subclasses should implement this method.**
391 """
392 pass
394 def _validateValue(self, value):
395 """Validate a value.
397 Parameters
398 ----------
399 value : object
400 The value being validated.
402 Raises
403 ------
404 TypeError
405 Raised if the value's type is incompatible with the field's
406 ``dtype``.
407 ValueError
408 Raised if the value is rejected by the ``check`` method.
409 """
410 if value is None: 410 ↛ 411line 410 didn't jump to line 411, because the condition on line 410 was never true
411 return
413 if not isinstance(value, self.dtype): 413 ↛ 414line 413 didn't jump to line 414, because the condition on line 413 was never true
414 msg = "Value %s is of incorrect type %s. Expected type %s" % \
415 (value, _typeStr(value), _typeStr(self.dtype))
416 raise TypeError(msg)
417 if self.check is not None and not self.check(value): 417 ↛ 418line 417 didn't jump to line 418, because the condition on line 417 was never true
418 msg = "Value %s is not a valid value" % str(value)
419 raise ValueError(msg)
421 def _collectImports(self, instance, imports):
422 """This function should call the _collectImports method on all config
423 objects the field may own, and union them with the supplied imports
424 set.
426 Parameters
427 ----------
428 instance : instance or subclass of `lsst.pex.config.Config`
429 A config object that has this field defined on it
430 imports : `set`
431 Set of python modules that need imported after persistence
432 """
433 pass
435 def save(self, outfile, instance):
436 """Save this field to a file (for internal use only).
438 Parameters
439 ----------
440 outfile : file-like object
441 A writeable field handle.
442 instance : `Config`
443 The `Config` instance that contains this field.
445 Notes
446 -----
447 This method is invoked by the `~lsst.pex.config.Config` object that
448 contains this field and should not be called directly.
450 The output consists of the documentation string
451 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
452 line is formatted as an assignment: ``{fullname}={value}``.
454 This output can be executed with Python.
455 """
456 value = self.__get__(instance)
457 fullname = _joinNamePath(instance._name, self.name)
459 if self.deprecated and value == self.default: 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true
460 return
462 # write full documentation string as comment lines
463 # (i.e. first character is #)
464 doc = "# " + str(self.doc).replace("\n", "\n# ")
465 if isinstance(value, float) and (math.isinf(value) or math.isnan(value)): 465 ↛ 467line 465 didn't jump to line 467, because the condition on line 465 was never true
466 # non-finite numbers need special care
467 outfile.write(u"{}\n{}=float('{!r}')\n\n".format(doc, fullname, value))
468 else:
469 outfile.write(u"{}\n{}={!r}\n\n".format(doc, fullname, value))
471 def toDict(self, instance):
472 """Convert the field value so that it can be set as the value of an
473 item in a `dict` (for internal use only).
475 Parameters
476 ----------
477 instance : `Config`
478 The `Config` that contains this field.
480 Returns
481 -------
482 value : object
483 The field's value. See *Notes*.
485 Notes
486 -----
487 This method invoked by the owning `~lsst.pex.config.Config` object and
488 should not be called directly.
490 Simple values are passed through. Complex data structures must be
491 manipulated. For example, a `~lsst.pex.config.Field` holding a
492 subconfig should, instead of the subconfig object, return a `dict`
493 where the keys are the field names in the subconfig, and the values are
494 the field values in the subconfig.
495 """
496 return self.__get__(instance)
498 def __get__(self, instance, owner=None, at=None, label="default"):
499 """Define how attribute access should occur on the Config instance
500 This is invoked by the owning config object and should not be called
501 directly
503 When the field attribute is accessed on a Config class object, it
504 returns the field object itself in order to allow inspection of
505 Config classes.
507 When the field attribute is access on a config instance, the actual
508 value described by the field (and held by the Config instance) is
509 returned.
510 """
511 if instance is None or not isinstance(instance, Config): 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true
512 return self
513 else:
514 return instance._storage[self.name]
516 def __set__(self, instance, value, at=None, label='assignment'):
517 """Set an attribute on the config instance.
519 Parameters
520 ----------
521 instance : `lsst.pex.config.Config`
522 The config instance that contains this field.
523 value : obj
524 Value to set on this field.
525 at : `list` of `lsst.pex.config.callStack.StackFrame`
526 The call stack (created by
527 `lsst.pex.config.callStack.getCallStack`).
528 label : `str`, optional
529 Event label for the history.
531 Notes
532 -----
533 This method is invoked by the owning `lsst.pex.config.Config` object
534 and should not be called directly.
536 Derived `~lsst.pex.config.Field` classes may need to override the
537 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
538 should follow the following rules:
540 - Do not allow modification of frozen configs.
541 - Validate the new value **before** modifying the field. Except if the
542 new value is `None`. `None` is special and no attempt should be made
543 to validate it until `lsst.pex.config.Config.validate` is called.
544 - Do not modify the `~lsst.pex.config.Config` instance to contain
545 invalid values.
546 - If the field is modified, update the history of the
547 `lsst.pex.config.field.Field` to reflect the changes.
549 In order to decrease the need to implement this method in derived
550 `~lsst.pex.config.Field` types, value validation is performed in the
551 `lsst.pex.config.Field._validateValue`. If only the validation step
552 differs in the derived `~lsst.pex.config.Field`, it is simpler to
553 implement `lsst.pex.config.Field._validateValue` than to reimplement
554 ``__set__``. More complicated behavior, however, may require
555 reimplementation.
556 """
557 if instance._frozen: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true
558 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
560 history = instance._history.setdefault(self.name, [])
561 if value is not None: 561 ↛ 568line 561 didn't jump to line 568, because the condition on line 561 was never false
562 value = _autocast(value, self.dtype)
563 try:
564 self._validateValue(value)
565 except BaseException as e:
566 raise FieldValidationError(self, instance, str(e))
568 instance._storage[self.name] = value
569 if at is None: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true
570 at = getCallStack()
571 history.append((value, at, label))
573 def __delete__(self, instance, at=None, label='deletion'):
574 """Delete an attribute from a `lsst.pex.config.Config` instance.
576 Parameters
577 ----------
578 instance : `lsst.pex.config.Config`
579 The config instance that contains this field.
580 at : `list` of `lsst.pex.config.callStack.StackFrame`
581 The call stack (created by
582 `lsst.pex.config.callStack.getCallStack`).
583 label : `str`, optional
584 Event label for the history.
586 Notes
587 -----
588 This is invoked by the owning `~lsst.pex.config.Config` object and
589 should not be called directly.
590 """
591 if at is None:
592 at = getCallStack()
593 self.__set__(instance, None, at=at, label=label)
595 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
596 """Compare a field (named `Field.name`) in two
597 `~lsst.pex.config.Config` instances for equality.
599 Parameters
600 ----------
601 instance1 : `lsst.pex.config.Config`
602 Left-hand side `Config` instance to compare.
603 instance2 : `lsst.pex.config.Config`
604 Right-hand side `Config` instance to compare.
605 shortcut : `bool`, optional
606 **Unused.**
607 rtol : `float`, optional
608 Relative tolerance for floating point comparisons.
609 atol : `float`, optional
610 Absolute tolerance for floating point comparisons.
611 output : callable, optional
612 A callable that takes a string, used (possibly repeatedly) to
613 report inequalities.
615 Notes
616 -----
617 This method must be overridden by more complex `Field` subclasses.
619 See also
620 --------
621 lsst.pex.config.compareScalars
622 """
623 v1 = getattr(instance1, self.name)
624 v2 = getattr(instance2, self.name)
625 name = getComparisonName(
626 _joinNamePath(instance1._name, self.name),
627 _joinNamePath(instance2._name, self.name)
628 )
629 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
632class RecordingImporter:
633 """Importer (for `sys.meta_path`) that records which modules are being
634 imported.
636 *This class does not do any importing itself.*
638 Examples
639 --------
640 Use this class as a context manager to ensure it is properly uninstalled
641 when done:
643 >>> with RecordingImporter() as importer:
644 ... # import stuff
645 ... import numpy as np
646 ... print("Imported: " + importer.getModules())
647 """
649 def __init__(self):
650 self._modules = set()
652 def __enter__(self):
653 self.origMetaPath = sys.meta_path
654 sys.meta_path = [self] + sys.meta_path
655 return self
657 def __exit__(self, *args):
658 self.uninstall()
659 return False # Don't suppress exceptions
661 def uninstall(self):
662 """Uninstall the importer.
663 """
664 sys.meta_path = self.origMetaPath
666 def find_module(self, fullname, path=None):
667 """Called as part of the ``import`` chain of events.
668 """
669 self._modules.add(fullname)
670 # Return None because we don't do any importing.
671 return None
673 def getModules(self):
674 """Get the set of modules that were imported.
676 Returns
677 -------
678 modules : `set` of `str`
679 Set of imported module names.
680 """
681 return self._modules
684class Config(metaclass=ConfigMeta):
685 """Base class for configuration (*config*) objects.
687 Notes
688 -----
689 A ``Config`` object will usually have several `~lsst.pex.config.Field`
690 instances as class attributes. These are used to define most of the base
691 class behavior.
693 ``Config`` implements a mapping API that provides many `dict`-like methods,
694 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and
695 `itervalues`. ``Config`` instances also support the ``in`` operator to
696 test if a field is in the config. Unlike a `dict`, ``Config`` classes are
697 not subscriptable. Instead, access individual fields as attributes of the
698 configuration instance.
700 Examples
701 --------
702 Config classes are subclasses of ``Config`` that have
703 `~lsst.pex.config.Field` instances (or instances of
704 `~lsst.pex.config.Field` subclasses) as class attributes:
706 >>> from lsst.pex.config import Config, Field, ListField
707 >>> class DemoConfig(Config):
708 ... intField = Field(doc="An integer field", dtype=int, default=42)
709 ... listField = ListField(doc="List of favorite beverages.", dtype=str,
710 ... default=['coffee', 'green tea', 'water'])
711 ...
712 >>> config = DemoConfig()
714 Configs support many `dict`-like APIs:
716 >>> config.keys()
717 ['intField', 'listField']
718 >>> 'intField' in config
719 True
721 Individual fields can be accessed as attributes of the configuration:
723 >>> config.intField
724 42
725 >>> config.listField.append('earl grey tea')
726 >>> print(config.listField)
727 ['coffee', 'green tea', 'water', 'earl grey tea']
728 """
730 def __iter__(self):
731 """Iterate over fields.
732 """
733 return self._fields.__iter__()
735 def keys(self):
736 """Get field names.
738 Returns
739 -------
740 names : `list`
741 List of `lsst.pex.config.Field` names.
743 See also
744 --------
745 lsst.pex.config.Config.iterkeys
746 """
747 return list(self._storage.keys())
749 def values(self):
750 """Get field values.
752 Returns
753 -------
754 values : `list`
755 List of field values.
757 See also
758 --------
759 lsst.pex.config.Config.itervalues
760 """
761 return list(self._storage.values())
763 def items(self):
764 """Get configurations as ``(field name, field value)`` pairs.
766 Returns
767 -------
768 items : `list`
769 List of tuples for each configuration. Tuple items are:
771 0. Field name.
772 1. Field value.
774 See also
775 --------
776 lsst.pex.config.Config.iteritems
777 """
778 return list(self._storage.items())
780 def iteritems(self):
781 """Iterate over (field name, field value) pairs.
783 Yields
784 ------
785 item : `tuple`
786 Tuple items are:
788 0. Field name.
789 1. Field value.
791 See also
792 --------
793 lsst.pex.config.Config.items
794 """
795 return iter(self._storage.items())
797 def itervalues(self):
798 """Iterate over field values.
800 Yields
801 ------
802 value : obj
803 A field value.
805 See also
806 --------
807 lsst.pex.config.Config.values
808 """
809 return iter(self.storage.values())
811 def iterkeys(self):
812 """Iterate over field names
814 Yields
815 ------
816 key : `str`
817 A field's key (attribute name).
819 See also
820 --------
821 lsst.pex.config.Config.values
822 """
823 return iter(self.storage.keys())
825 def __contains__(self, name):
826 """!Return True if the specified field exists in this config
828 @param[in] name field name to test for
829 """
830 return self._storage.__contains__(name)
832 def __new__(cls, *args, **kw):
833 """Allocate a new `lsst.pex.config.Config` object.
835 In order to ensure that all Config object are always in a proper state
836 when handed to users or to derived `~lsst.pex.config.Config` classes,
837 some attributes are handled at allocation time rather than at
838 initialization.
840 This ensures that even if a derived `~lsst.pex.config.Config` class
841 implements ``__init__``, its author does not need to be concerned about
842 when or even the base ``Config.__init__`` should be called.
843 """
844 name = kw.pop("__name", None)
845 at = kw.pop("__at", getCallStack())
846 # remove __label and ignore it
847 kw.pop("__label", "default")
849 instance = object.__new__(cls)
850 instance._frozen = False
851 instance._name = name
852 instance._storage = {}
853 instance._history = {}
854 instance._imports = set()
855 # load up defaults
856 for field in instance._fields.values():
857 instance._history[field.name] = []
858 field.__set__(instance, field.default, at=at + [field.source], label="default")
859 # set custom default-overides
860 instance.setDefaults()
861 # set constructor overides
862 instance.update(__at=at, **kw)
863 return instance
865 def __reduce__(self):
866 """Reduction for pickling (function with arguments to reproduce).
868 We need to condense and reconstitute the `~lsst.pex.config.Config`,
869 since it may contain lambdas (as the ``check`` elements) that cannot
870 be pickled.
871 """
872 # The stream must be in characters to match the API but pickle
873 # requires bytes
874 stream = io.StringIO()
875 self.saveToStream(stream)
876 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
878 def setDefaults(self):
879 """Subclass hook for computing defaults.
881 Notes
882 -----
883 Derived `~lsst.pex.config.Config` classes that must compute defaults
884 rather than using the `~lsst.pex.config.Field` instances's defaults
885 should do so here. To correctly use inherited defaults,
886 implementations of ``setDefaults`` must call their base class's
887 ``setDefaults``.
888 """
889 pass
891 def update(self, **kw):
892 """Update values of fields specified by the keyword arguments.
894 Parameters
895 ----------
896 kw
897 Keywords are configuration field names. Values are configuration
898 field values.
900 Notes
901 -----
902 The ``__at`` and ``__label`` keyword arguments are special internal
903 keywords. They are used to strip out any internal steps from the
904 history tracebacks of the config. Do not modify these keywords to
905 subvert a `~lsst.pex.config.Config` instance's history.
907 Examples
908 --------
909 This is a config with three fields:
911 >>> from lsst.pex.config import Config, Field
912 >>> class DemoConfig(Config):
913 ... fieldA = Field(doc='Field A', dtype=int, default=42)
914 ... fieldB = Field(doc='Field B', dtype=bool, default=True)
915 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
916 ...
917 >>> config = DemoConfig()
919 These are the default values of each field:
921 >>> for name, value in config.iteritems():
922 ... print(f"{name}: {value}")
923 ...
924 fieldA: 42
925 fieldB: True
926 fieldC: 'Hello world'
928 Using this method to update ``fieldA`` and ``fieldC``:
930 >>> config.update(fieldA=13, fieldC='Updated!')
932 Now the values of each field are:
934 >>> for name, value in config.iteritems():
935 ... print(f"{name}: {value}")
936 ...
937 fieldA: 13
938 fieldB: True
939 fieldC: 'Updated!'
940 """
941 at = kw.pop("__at", getCallStack())
942 label = kw.pop("__label", "update")
944 for name, value in kw.items():
945 try:
946 field = self._fields[name]
947 field.__set__(self, value, at=at, label=label)
948 except KeyError:
949 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
951 def load(self, filename, root="config"):
952 """Modify this config in place by executing the Python code in a
953 configuration file.
955 Parameters
956 ----------
957 filename : `str`
958 Name of the configuration file. A configuration file is Python
959 module.
960 root : `str`, optional
961 Name of the variable in file that refers to the config being
962 overridden.
964 For example, the value of root is ``"config"`` and the file
965 contains::
967 config.myField = 5
969 Then this config's field ``myField`` is set to ``5``.
971 **Deprecated:** For backwards compatibility, older config files
972 that use ``root="root"`` instead of ``root="config"`` will be
973 loaded with a warning printed to `sys.stderr`. This feature will be
974 removed at some point.
976 See also
977 --------
978 lsst.pex.config.Config.loadFromStream
979 lsst.pex.config.Config.save
980 lsst.pex.config.Config.saveFromStream
981 """
982 with open(filename, "r") as f:
983 code = compile(f.read(), filename=filename, mode="exec")
984 self.loadFromStream(stream=code, root=root, filename=filename)
986 def loadFromStream(self, stream, root="config", filename=None):
987 """Modify this Config in place by executing the Python code in the
988 provided stream.
990 Parameters
991 ----------
992 stream : file-like object, `str`, or compiled string
993 Stream containing configuration override code.
994 root : `str`, optional
995 Name of the variable in file that refers to the config being
996 overridden.
998 For example, the value of root is ``"config"`` and the file
999 contains::
1001 config.myField = 5
1003 Then this config's field ``myField`` is set to ``5``.
1005 **Deprecated:** For backwards compatibility, older config files
1006 that use ``root="root"`` instead of ``root="config"`` will be
1007 loaded with a warning printed to `sys.stderr`. This feature will be
1008 removed at some point.
1009 filename : `str`, optional
1010 Name of the configuration file, or `None` if unknown or contained
1011 in the stream. Used for error reporting.
1013 See also
1014 --------
1015 lsst.pex.config.Config.load
1016 lsst.pex.config.Config.save
1017 lsst.pex.config.Config.saveFromStream
1018 """
1019 with RecordingImporter() as importer:
1020 globals = {"__file__": filename}
1021 try:
1022 local = {root: self}
1023 exec(stream, globals, local)
1024 except NameError as e:
1025 if root == "config" and "root" in e.args[0]:
1026 if filename is None:
1027 # try to determine the file name; a compiled string
1028 # has attribute "co_filename",
1029 # an open file has attribute "name", else give up
1030 filename = getattr(stream, "co_filename", None)
1031 if filename is None:
1032 filename = getattr(stream, "name", "?")
1033 print(f"Config override file {filename!r}"
1034 " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
1035 local = {"root": self}
1036 exec(stream, globals, local)
1037 else:
1038 raise
1040 self._imports.update(importer.getModules())
1042 def save(self, filename, root="config"):
1043 """Save a Python script to the named file, which, when loaded,
1044 reproduces this config.
1046 Parameters
1047 ----------
1048 filename : `str`
1049 Desination filename of this configuration.
1050 root : `str`, optional
1051 Name to use for the root config variable. The same value must be
1052 used when loading (see `lsst.pex.config.Config.load`).
1054 See also
1055 --------
1056 lsst.pex.config.Config.saveToStream
1057 lsst.pex.config.Config.load
1058 lsst.pex.config.Config.loadFromStream
1059 """
1060 d = os.path.dirname(filename)
1061 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1062 self.saveToStream(outfile, root)
1063 # tempfile is hardcoded to create files with mode '0600'
1064 # for an explantion of these antics see:
1065 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1066 umask = os.umask(0o077)
1067 os.umask(umask)
1068 os.chmod(outfile.name, (~umask & 0o666))
1069 # chmod before the move so we get quasi-atomic behavior if the
1070 # source and dest. are on the same filesystem.
1071 # os.rename may not work across filesystems
1072 shutil.move(outfile.name, filename)
1074 def saveToStream(self, outfile, root="config", skipImports=False):
1075 """Save a configuration file to a stream, which, when loaded,
1076 reproduces this config.
1078 Parameters
1079 ----------
1080 outfile : file-like object
1081 Destination file object write the config into. Accepts strings not
1082 bytes.
1083 root
1084 Name to use for the root config variable. The same value must be
1085 used when loading (see `lsst.pex.config.Config.load`).
1086 skipImports : `bool`, optional
1087 If `True` then do not include ``import`` statements in output,
1088 this is to support human-oriented output from ``pipetask`` where
1089 additional clutter is not useful.
1091 See also
1092 --------
1093 lsst.pex.config.Config.save
1094 lsst.pex.config.Config.load
1095 lsst.pex.config.Config.loadFromStream
1096 """
1097 tmp = self._name
1098 self._rename(root)
1099 try:
1100 if not skipImports: 1100 ↛ 1112line 1100 didn't jump to line 1112, because the condition on line 1100 was never false
1101 self._collectImports()
1102 # Remove self from the set, as it is handled explicitly below
1103 self._imports.remove(self.__module__)
1104 configType = type(self)
1105 typeString = _typeStr(configType)
1106 outfile.write(f"import {configType.__module__}\n")
1107 outfile.write(f"assert type({root})=={typeString}, 'config is of type %s.%s instead of "
1108 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n")
1109 for imp in self._imports: 1109 ↛ 1110line 1109 didn't jump to line 1110, because the loop on line 1109 never started
1110 if imp in sys.modules and sys.modules[imp] is not None:
1111 outfile.write(u"import {}\n".format(imp))
1112 self._save(outfile)
1113 finally:
1114 self._rename(tmp)
1116 def freeze(self):
1117 """Make this config, and all subconfigs, read-only.
1118 """
1119 self._frozen = True
1120 for field in self._fields.values():
1121 field.freeze(self)
1123 def _save(self, outfile):
1124 """Save this config to an open stream object.
1126 Parameters
1127 ----------
1128 outfile : file-like object
1129 Destination file object write the config into. Accepts strings not
1130 bytes.
1131 """
1132 for field in self._fields.values():
1133 field.save(outfile, self)
1135 def _collectImports(self):
1136 """Adds module containing self to the list of things to import and
1137 then loops over all the fields in the config calling a corresponding
1138 collect method. The field method will call _collectImports on any
1139 configs it may own and return the set of things to import. This
1140 returned set will be merged with the set of imports for this config
1141 class.
1142 """
1143 self._imports.add(self.__module__)
1144 for name, field in self._fields.items():
1145 field._collectImports(self, self._imports)
1147 def toDict(self):
1148 """Make a dictionary of field names and their values.
1150 Returns
1151 -------
1152 dict_ : `dict`
1153 Dictionary with keys that are `~lsst.pex.config.Field` names.
1154 Values are `~lsst.pex.config.Field` values.
1156 See also
1157 --------
1158 lsst.pex.config.Field.toDict
1160 Notes
1161 -----
1162 This method uses the `~lsst.pex.config.Field.toDict` method of
1163 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1164 implement a ``toDict`` method for *this* method to work.
1165 """
1166 dict_ = {}
1167 for name, field in self._fields.items():
1168 dict_[name] = field.toDict(self)
1169 return dict_
1171 def names(self):
1172 """Get all the field names in the config, recursively.
1174 Returns
1175 -------
1176 names : `list` of `str`
1177 Field names.
1178 """
1179 #
1180 # Rather than sort out the recursion all over again use the
1181 # pre-existing saveToStream()
1182 #
1183 with io.StringIO() as strFd:
1184 self.saveToStream(strFd, "config")
1185 contents = strFd.getvalue()
1186 strFd.close()
1187 #
1188 # Pull the names out of the dumped config
1189 #
1190 keys = []
1191 for line in contents.split("\n"):
1192 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1193 continue
1195 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1196 if mat:
1197 keys.append(mat.group(1))
1199 return keys
1201 def _rename(self, name):
1202 """Rename this config object in its parent `~lsst.pex.config.Config`.
1204 Parameters
1205 ----------
1206 name : `str`
1207 New name for this config in its parent `~lsst.pex.config.Config`.
1209 Notes
1210 -----
1211 This method uses the `~lsst.pex.config.Field.rename` method of
1212 individual `lsst.pex.config.Field` instances.
1213 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1214 method for *this* method to work.
1216 See also
1217 --------
1218 lsst.pex.config.Field.rename
1219 """
1220 self._name = name
1221 for field in self._fields.values():
1222 field.rename(self)
1224 def validate(self):
1225 """Validate the Config, raising an exception if invalid.
1227 Raises
1228 ------
1229 lsst.pex.config.FieldValidationError
1230 Raised if verification fails.
1232 Notes
1233 -----
1234 The base class implementation performs type checks on all fields by
1235 calling their `~lsst.pex.config.Field.validate` methods.
1237 Complex single-field validation can be defined by deriving new Field
1238 types. For convenience, some derived `lsst.pex.config.Field`-types
1239 (`~lsst.pex.config.ConfigField` and
1240 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config`
1241 that handle recursing into subconfigs.
1243 Inter-field relationships should only be checked in derived
1244 `~lsst.pex.config.Config` classes after calling this method, and base
1245 validation is complete.
1246 """
1247 for field in self._fields.values():
1248 field.validate(self)
1250 def formatHistory(self, name, **kwargs):
1251 """Format a configuration field's history to a human-readable format.
1253 Parameters
1254 ----------
1255 name : `str`
1256 Name of a `~lsst.pex.config.Field` in this config.
1257 kwargs
1258 Keyword arguments passed to `lsst.pex.config.history.format`.
1260 Returns
1261 -------
1262 history : `str`
1263 A string containing the formatted history.
1265 See also
1266 --------
1267 lsst.pex.config.history.format
1268 """
1269 import lsst.pex.config.history as pexHist
1270 return pexHist.format(self, name, **kwargs)
1272 history = property(lambda x: x._history) 1272 ↛ exitline 1272 didn't run the lambda on line 1272
1273 """Read-only history.
1274 """
1276 def __setattr__(self, attr, value, at=None, label="assignment"):
1277 """Set an attribute (such as a field's value).
1279 Notes
1280 -----
1281 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1282 locked such that no additional attributes nor properties may be added
1283 to them dynamically.
1285 Although this is not the standard Python behavior, it helps to protect
1286 users from accidentally mispelling a field name, or trying to set a
1287 non-existent field.
1288 """
1289 if attr in self._fields:
1290 if self._fields[attr].deprecated is not None: 1290 ↛ 1291line 1290 didn't jump to line 1291, because the condition on line 1290 was never true
1291 fullname = _joinNamePath(self._name, self._fields[attr].name)
1292 warnings.warn(f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1293 FutureWarning, stacklevel=2)
1294 if at is None: 1294 ↛ 1297line 1294 didn't jump to line 1297, because the condition on line 1294 was never false
1295 at = getCallStack()
1296 # This allows Field descriptors to work.
1297 self._fields[attr].__set__(self, value, at=at, label=label)
1298 elif hasattr(getattr(self.__class__, attr, None), '__set__'): 1298 ↛ 1300line 1298 didn't jump to line 1300, because the condition on line 1298 was never true
1299 # This allows properties and other non-Field descriptors to work.
1300 return object.__setattr__(self, attr, value)
1301 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1301 ↛ 1306line 1301 didn't jump to line 1306, because the condition on line 1301 was never false
1302 # This allows specific private attributes to work.
1303 self.__dict__[attr] = value
1304 else:
1305 # We throw everything else.
1306 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr))
1308 def __delattr__(self, attr, at=None, label="deletion"):
1309 if attr in self._fields:
1310 if at is None:
1311 at = getCallStack()
1312 self._fields[attr].__delete__(self, at=at, label=label)
1313 else:
1314 object.__delattr__(self, attr)
1316 def __eq__(self, other):
1317 if type(other) == type(self): 1317 ↛ 1318line 1317 didn't jump to line 1318, because the condition on line 1317 was never true
1318 for name in self._fields:
1319 thisValue = getattr(self, name)
1320 otherValue = getattr(other, name)
1321 if isinstance(thisValue, float) and math.isnan(thisValue):
1322 if not math.isnan(otherValue):
1323 return False
1324 elif thisValue != otherValue:
1325 return False
1326 return True
1327 return False
1329 def __ne__(self, other):
1330 return not self.__eq__(other)
1332 def __str__(self):
1333 return str(self.toDict())
1335 def __repr__(self):
1336 return "%s(%s)" % (
1337 _typeStr(self),
1338 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None)
1339 )
1341 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
1342 """Compare this configuration to another `~lsst.pex.config.Config` for
1343 equality.
1345 Parameters
1346 ----------
1347 other : `lsst.pex.config.Config`
1348 Other `~lsst.pex.config.Config` object to compare against this
1349 config.
1350 shortcut : `bool`, optional
1351 If `True`, return as soon as an inequality is found. Default is
1352 `True`.
1353 rtol : `float`, optional
1354 Relative tolerance for floating point comparisons.
1355 atol : `float`, optional
1356 Absolute tolerance for floating point comparisons.
1357 output : callable, optional
1358 A callable that takes a string, used (possibly repeatedly) to
1359 report inequalities.
1361 Returns
1362 -------
1363 isEqual : `bool`
1364 `True` when the two `lsst.pex.config.Config` instances are equal.
1365 `False` if there is an inequality.
1367 See also
1368 --------
1369 lsst.pex.config.compareConfigs
1371 Notes
1372 -----
1373 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1374 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1375 are not considered by this method.
1377 Floating point comparisons are performed by `numpy.allclose`.
1378 """
1379 name1 = self._name if self._name is not None else "config"
1380 name2 = other._name if other._name is not None else "config"
1381 name = getComparisonName(name1, name2)
1382 return compareConfigs(name, self, other, shortcut=shortcut,
1383 rtol=rtol, atol=atol, output=output)
1386def unreduceConfig(cls, stream):
1387 """Create a `~lsst.pex.config.Config` from a stream.
1389 Parameters
1390 ----------
1391 cls : `lsst.pex.config.Config`-type
1392 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1393 with configurations in the ``stream``.
1394 stream : file-like object, `str`, or compiled string
1395 Stream containing configuration override code.
1397 Returns
1398 -------
1399 config : `lsst.pex.config.Config`
1400 Config instance.
1402 See also
1403 --------
1404 lsst.pex.config.Config.loadFromStream
1405 """
1406 config = cls()
1407 config.loadFromStream(stream)
1408 return config