Coverage for python/lsst/daf/butler/core/config.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Configuration control."""
26__all__ = ("Config", "ConfigSubset")
28import collections
29import copy
30import logging
31import pprint
32import os
33import yaml
34import sys
35from yaml.representer import Representer
36import io
37from typing import Sequence, Optional, ClassVar, IO, Union
39from lsst.utils import doImport
40from ._butlerUri import ButlerURI
42yaml.add_representer(collections.defaultdict, Representer.represent_dict)
45# Config module logger
46log = logging.getLogger(__name__)
48# PATH-like environment variable to use for defaults.
49CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH"
51try:
52 yamlLoader = yaml.CSafeLoader
53except AttributeError:
54 # Not all installations have the C library
55 yamlLoader = yaml.SafeLoader
58class Loader(yamlLoader):
59 """YAML Loader that supports file include directives
61 Uses ``!include`` directive in a YAML file to point to another
62 YAML file to be included. The path in the include directive is relative
63 to the file containing that directive.
65 storageClasses: !include storageClasses.yaml
67 Examples
68 --------
69 >>> with open("document.yaml", "r") as f:
70 data = yaml.load(f, Loader=Loader)
72 Notes
73 -----
74 See https://davidchall.github.io/yaml-includes.html
75 """
77 def __init__(self, stream):
78 super().__init__(stream)
79 # if this is a string and not a stream we may well lack a name
80 try:
81 self._root = ButlerURI(stream.name)
82 except AttributeError:
83 # No choice but to assume a local filesystem
84 self._root = ButlerURI("no-file.yaml")
85 Loader.add_constructor("!include", Loader.include)
87 def include(self, node):
88 if isinstance(node, yaml.ScalarNode):
89 return self.extractFile(self.construct_scalar(node))
91 elif isinstance(node, yaml.SequenceNode):
92 result = []
93 for filename in self.construct_sequence(node):
94 result.append(self.extractFile(filename))
95 return result
97 elif isinstance(node, yaml.MappingNode):
98 result = {}
99 for k, v in self.construct_mapping(node).items():
100 result[k] = self.extractFile(v)
101 return result
103 else:
104 print("Error:: unrecognised node type in !include statement", file=sys.stderr)
105 raise yaml.constructor.ConstructorError
107 def extractFile(self, filename):
108 # It is possible for the !include to point to an explicit URI
109 # instead of a relative URI, therefore we first see if it is
110 # scheme-less or not. If it has a scheme we use it directly
111 # if it is scheme-less we use it relative to the file root.
112 requesteduri = ButlerURI(filename, forceAbsolute=False)
114 if requesteduri.scheme:
115 fileuri = requesteduri
116 else:
117 fileuri = copy.copy(self._root)
118 fileuri.updateFile(filename)
120 log.debug("Opening YAML file via !include: %s", fileuri)
122 # Read all the data from the resource
123 data = fileuri.read()
125 # Store the bytes into a BytesIO so we can attach a .name
126 stream = io.BytesIO(data)
127 stream.name = fileuri.geturl()
128 return yaml.load(stream, Loader)
131class Config(collections.abc.MutableMapping):
132 r"""Implements a datatype that is used by `Butler` for configuration
133 parameters.
135 It is essentially a `dict` with key/value pairs, including nested dicts
136 (as values). In fact, it can be initialized with a `dict`.
137 This is explained next:
139 Config extends the `dict` api so that hierarchical values may be accessed
140 with delimited notation or as a tuple. If a string is given the delimiter
141 is picked up from the first character in that string. For example,
142 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``,
143 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome.
144 If the first character is alphanumeric, no delimiter will be used.
145 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``.
146 Unicode characters can be used as the delimiter for distinctiveness if
147 required.
149 If a key in the hierarchy starts with a non-alphanumeric character care
150 should be used to ensure that either the tuple interface is used or
151 a distinct delimiter is always given in string form.
153 Finally, the delimiter can be escaped if it is part of a key and also
154 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in
155 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is
156 always better to use a different delimiter in these cases.
158 Note that adding a multi-level key implicitly creates any nesting levels
159 that do not exist, but removing multi-level keys does not automatically
160 remove empty nesting levels. As a result:
162 >>> c = Config()
163 >>> c[".a.b"] = 1
164 >>> del c[".a.b"]
165 >>> c["a"]
166 Config({'a': {}})
168 Storage formats supported:
170 - yaml: read and write is supported.
173 Parameters
174 ----------
175 other : `str` or `Config` or `dict` or `ButlerURI`
176 Other source of configuration, can be:
178 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end
179 with ".yaml".
180 - (`Config`) Copies the other Config's values into this one.
181 - (`dict`) Copies the values from the dict into this Config.
183 If `None` is provided an empty `Config` will be created.
184 """
186 _D: ClassVar[str] = "→"
187 """Default internal delimiter to use for components in the hierarchy when
188 constructing keys for external use (see `Config.names()`)."""
190 includeKey: ClassVar[str] = "includeConfigs"
191 """Key used to indicate that another config should be included at this
192 part of the hierarchy."""
194 resourcesPackage: str = "lsst.daf.butler"
195 """Package to search for default configuration data. The resources
196 themselves will be within a ``configs`` resource hierarchy."""
198 def __init__(self, other=None):
199 self._data = {}
200 self.configFile = None
202 if other is None:
203 return
205 if isinstance(other, Config):
206 self._data = copy.deepcopy(other._data)
207 self.configFile = other.configFile
208 elif isinstance(other, collections.abc.Mapping):
209 self.update(other)
210 elif isinstance(other, (str, ButlerURI)):
211 # if other is a string, assume it is a file path/URI
212 self.__initFromUri(other)
213 self._processExplicitIncludes()
214 else:
215 # if the config specified by other could not be recognized raise
216 # a runtime error.
217 raise RuntimeError(f"A Config could not be loaded from other: {other}")
219 def ppprint(self):
220 """helper function for debugging, prints a config out in a readable
221 way in the debugger.
223 use: pdb> print(myConfigObject.ppprint())
225 Returns
226 -------
227 s : `str`
228 A prettyprint formatted string representing the config
229 """
230 return pprint.pformat(self._data, indent=2, width=1)
232 def __repr__(self):
233 return f"{type(self).__name__}({self._data!r})"
235 def __str__(self):
236 return self.ppprint()
238 def __len__(self):
239 return len(self._data)
241 def __iter__(self):
242 return iter(self._data)
244 def copy(self):
245 return type(self)(self)
247 @classmethod
248 def fromYaml(cls, string: str) -> Config:
249 """Create a new Config instance from a YAML string.
251 Parameters
252 ----------
253 string : `str`
254 String containing content in YAML format
256 Returns
257 -------
258 c : `Config`
259 Newly-constructed Config.
260 """
261 new_config = cls().__initFromYaml(string)
262 new_config._processExplicitIncludes()
263 return new_config
265 def __initFromUri(self, path: str) -> None:
266 """Load a file from a path or an URI.
268 Parameters
269 ----------
270 path : `str`
271 Path or a URI to a persisted config file.
272 """
273 uri = ButlerURI(path)
274 if uri.getExtension() == ".yaml":
275 log.debug("Opening YAML config file: %s", uri.geturl())
276 content = uri.read()
277 # Use a stream so we can name it
278 stream = io.BytesIO(content)
279 stream.name = uri.geturl()
280 self.__initFromYaml(stream)
281 else:
282 raise RuntimeError(f"Unhandled config file type: {uri}")
283 self.configFile = uri
285 def __initFromYaml(self, stream):
286 """Loads a YAML config from any readable stream that contains one.
288 Parameters
289 ----------
290 stream: `IO` or `str`
291 Stream to pass to the YAML loader. Accepts anything that
292 `yaml.load` accepts. This can include a string as well as an
293 IO stream.
295 Raises
296 ------
297 yaml.YAMLError
298 If there is an error loading the file.
299 """
300 content = yaml.load(stream, Loader=Loader)
301 if content is None:
302 content = {}
303 self._data = content
304 return self
306 def _processExplicitIncludes(self):
307 """Scan through the configuration searching for the special
308 includeConfigs directive and process the includes."""
310 # Search paths for config files
311 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)]
312 if self.configFile is not None:
313 if isinstance(self.configFile, ButlerURI):
314 configDir = self.configFile.dirname()
315 else:
316 raise RuntimeError(f"Unexpected type for config file: {self.configFile}")
317 searchPaths.append(configDir)
319 # Ensure we know what delimiter to use
320 names = self.nameTuples()
321 for path in names:
322 if path[-1] == self.includeKey:
324 log.debug("Processing file include directive at %s", self._D + self._D.join(path))
325 basePath = path[:-1]
327 # Extract the includes and then delete them from the config
328 includes = self[path]
329 del self[path]
331 # Be consistent and convert to a list
332 if not isinstance(includes, list):
333 includes = [includes]
335 # Read each file assuming it is a reference to a file
336 # The file can be relative to config file or cwd
337 # ConfigSubset search paths are not used
338 subConfigs = []
339 for fileName in includes:
340 # Expand any shell variables -- this could be URI
341 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False)
342 found = None
343 if fileName.isabs():
344 found = fileName
345 else:
346 for dir in searchPaths:
347 if isinstance(dir, ButlerURI):
348 specific = dir.join(fileName.path)
349 # Remote resource check might be expensive
350 if specific.exists():
351 found = specific
352 else:
353 log.warning("Do not understand search path entry '%s' of type %s",
354 dir, type(dir).__name__)
355 if not found:
356 raise RuntimeError(f"Unable to find referenced include file: {fileName}")
358 # Read the referenced Config as a Config
359 subConfigs.append(type(self)(found))
361 # Now we need to merge these sub configs with the current
362 # information that was present in this node in the config
363 # tree with precedence given to the explicit values
364 newConfig = subConfigs.pop(0)
365 for sc in subConfigs:
366 newConfig.update(sc)
368 # Explicit values take precedence
369 if not basePath:
370 # This is an include at the root config
371 newConfig.update(self)
372 # Replace the current config
373 self._data = newConfig._data
374 else:
375 newConfig.update(self[basePath])
376 # And reattach to the base config
377 self[basePath] = newConfig
379 @staticmethod
380 def _splitIntoKeys(key):
381 r"""Split the argument for get/set/in into a hierarchical list.
383 Parameters
384 ----------
385 key : `str` or iterable
386 Argument given to get/set/in. If an iterable is provided it will
387 be converted to a list. If the first character of the string
388 is not an alphanumeric character then it will be used as the
389 delimiter for the purposes of splitting the remainder of the
390 string. If the delimiter is also in one of the keys then it
391 can be escaped using ``\``. There is no default delimiter.
393 Returns
394 -------
395 keys : `list`
396 Hierarchical keys as a `list`.
397 """
398 if isinstance(key, str):
399 if not key[0].isalnum():
400 d = key[0]
401 key = key[1:]
402 else:
403 return [key, ]
404 escaped = f"\\{d}"
405 temp = None
406 if escaped in key:
407 # Complain at the attempt to escape the escape
408 doubled = fr"\{escaped}"
409 if doubled in key:
410 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})"
411 " is not yet supported.")
412 # Replace with a character that won't be in the string
413 temp = "\r"
414 if temp in key or d == temp:
415 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as"
416 " delimiter if escaping the delimiter")
417 key = key.replace(escaped, temp)
418 hierarchy = key.split(d)
419 if temp:
420 hierarchy = [h.replace(temp, d) for h in hierarchy]
421 return hierarchy
422 elif isinstance(key, collections.abc.Iterable):
423 return list(key)
424 else:
425 # Not sure what this is so try it anyway
426 return [key, ]
428 def _getKeyHierarchy(self, name):
429 """Retrieve the key hierarchy for accessing the Config
431 Parameters
432 ----------
433 name : `str` or `tuple`
434 Delimited string or `tuple` of hierarchical keys.
436 Returns
437 -------
438 hierarchy : `list` of `str`
439 Hierarchy to use as a `list`. If the name is available directly
440 as a key in the Config it will be used regardless of the presence
441 of any nominal delimiter.
442 """
443 if name in self._data:
444 keys = [name, ]
445 else:
446 keys = self._splitIntoKeys(name)
447 return keys
449 def _findInHierarchy(self, keys, create=False):
450 """Look for hierarchy of keys in Config
452 Parameters
453 ----------
454 keys : `list` or `tuple`
455 Keys to search in hierarchy.
456 create : `bool`, optional
457 If `True`, if a part of the hierarchy does not exist, insert an
458 empty `dict` into the hierarchy.
460 Returns
461 -------
462 hierarchy : `list`
463 List of the value corresponding to each key in the supplied
464 hierarchy. Only keys that exist in the hierarchy will have
465 a value.
466 complete : `bool`
467 `True` if the full hierarchy exists and the final element
468 in ``hierarchy`` is the value of relevant value.
469 """
470 d = self._data
472 def checkNextItem(k, d, create):
473 """See if k is in d and if it is return the new child"""
474 nextVal = None
475 isThere = False
476 if d is None:
477 # We have gone past the end of the hierarchy
478 pass
479 elif isinstance(d, collections.abc.Sequence):
480 # Check sequence first because for lists
481 # __contains__ checks whether value is found in list
482 # not whether the index exists in list. When we traverse
483 # the hierarchy we are interested in the index.
484 try:
485 nextVal = d[int(k)]
486 isThere = True
487 except IndexError:
488 pass
489 except ValueError:
490 isThere = k in d
491 elif k in d:
492 nextVal = d[k]
493 isThere = True
494 elif create:
495 d[k] = {}
496 nextVal = d[k]
497 isThere = True
498 return nextVal, isThere
500 hierarchy = []
501 complete = True
502 for k in keys:
503 d, isThere = checkNextItem(k, d, create)
504 if isThere:
505 hierarchy.append(d)
506 else:
507 complete = False
508 break
510 return hierarchy, complete
512 def __getitem__(self, name):
513 # Override the split for the simple case where there is an exact
514 # match. This allows `Config.items()` to work via a simple
515 # __iter__ implementation that returns top level keys of
516 # self._data.
517 keys = self._getKeyHierarchy(name)
519 hierarchy, complete = self._findInHierarchy(keys)
520 if not complete:
521 raise KeyError(f"{name} not found")
522 data = hierarchy[-1]
524 if isinstance(data, collections.abc.Mapping):
525 data = Config(data)
526 # Ensure that child configs inherit the parent internal delimiter
527 if self._D != Config._D:
528 data._D = self._D
529 return data
531 def __setitem__(self, name, value):
532 keys = self._getKeyHierarchy(name)
533 last = keys.pop()
534 if isinstance(value, Config):
535 value = copy.deepcopy(value._data)
537 hierarchy, complete = self._findInHierarchy(keys, create=True)
538 if hierarchy:
539 data = hierarchy[-1]
540 else:
541 data = self._data
543 try:
544 data[last] = value
545 except TypeError:
546 data[int(last)] = value
548 def __contains__(self, key):
549 keys = self._getKeyHierarchy(key)
550 hierarchy, complete = self._findInHierarchy(keys)
551 return complete
553 def __delitem__(self, key):
554 keys = self._getKeyHierarchy(key)
555 last = keys.pop()
556 hierarchy, complete = self._findInHierarchy(keys)
557 if complete:
558 if hierarchy:
559 data = hierarchy[-1]
560 else:
561 data = self._data
562 del data[last]
563 else:
564 raise KeyError(f"{key} not found in Config")
566 def update(self, other):
567 """Like dict.update, but will add or modify keys in nested dicts,
568 instead of overwriting the nested dict entirely.
570 For example, for the given code:
571 foo = {"a": {"b": 1}}
572 foo.update({"a": {"c": 2}})
574 Parameters
575 ----------
576 other : `dict` or `Config`
577 Source of configuration:
579 - If foo is a dict, then after the update foo == {"a": {"c": 2}}
580 - But if foo is a Config, then after the update
581 foo == {"a": {"b": 1, "c": 2}}
582 """
583 def doUpdate(d, u):
584 if not isinstance(u, collections.abc.Mapping) or \
585 not isinstance(d, collections.abc.Mapping):
586 raise RuntimeError("Only call update with Mapping, not {}".format(type(d)))
587 for k, v in u.items():
588 if isinstance(v, collections.abc.Mapping):
589 d[k] = doUpdate(d.get(k, {}), v)
590 else:
591 d[k] = v
592 return d
593 doUpdate(self._data, other)
595 def merge(self, other):
596 """Like Config.update, but will add keys & values from other that
597 DO NOT EXIST in self.
599 Keys and values that already exist in self will NOT be overwritten.
601 Parameters
602 ----------
603 other : `dict` or `Config`
604 Source of configuration:
605 """
606 otherCopy = copy.deepcopy(other)
607 otherCopy.update(self)
608 self._data = otherCopy._data
610 def nameTuples(self, topLevelOnly=False):
611 """Get tuples representing the name hierarchies of all keys.
613 The tuples returned from this method are guaranteed to be usable
614 to access items in the configuration object.
616 Parameters
617 ----------
618 topLevelOnly : `bool`, optional
619 If False, the default, a full hierarchy of names is returned.
620 If True, only the top level are returned.
622 Returns
623 -------
624 names : `list` of `tuple` of `str`
625 List of all names present in the `Config` where each element
626 in the list is a `tuple` of strings representing the hierarchy.
627 """
628 if topLevelOnly:
629 return list((k,) for k in self)
631 def getKeysAsTuples(d, keys, base):
632 if isinstance(d, collections.abc.Sequence):
633 theseKeys = range(len(d))
634 else:
635 theseKeys = d.keys()
636 for key in theseKeys:
637 val = d[key]
638 levelKey = base + (key,) if base is not None else (key,)
639 keys.append(levelKey)
640 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \
641 and not isinstance(val, str):
642 getKeysAsTuples(val, keys, levelKey)
643 keys = []
644 getKeysAsTuples(self._data, keys, None)
645 return keys
647 def names(self, topLevelOnly=False, delimiter=None):
648 """Get a delimited name of all the keys in the hierarchy.
650 The values returned from this method are guaranteed to be usable
651 to access items in the configuration object.
653 Parameters
654 ----------
655 topLevelOnly : `bool`, optional
656 If False, the default, a full hierarchy of names is returned.
657 If True, only the top level are returned.
658 delimiter : `str`, optional
659 Delimiter to use when forming the keys. If the delimiter is
660 present in any of the keys, it will be escaped in the returned
661 names. If `None` given a delimiter will be automatically provided.
662 The delimiter can not be alphanumeric.
664 Returns
665 -------
666 names : `list` of `str`
667 List of all names present in the `Config`.
669 Notes
670 -----
671 This is different than the built-in method `dict.keys`, which will
672 return only the first level keys.
674 Raises
675 ------
676 ValueError:
677 The supplied delimiter is alphanumeric.
678 """
679 if topLevelOnly:
680 return list(self.keys())
682 # Get all the tuples of hierarchical keys
683 nameTuples = self.nameTuples()
685 if delimiter is not None and delimiter.isalnum():
686 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.")
688 if delimiter is None:
689 # Start with something, and ensure it does not need to be
690 # escaped (it is much easier to understand if not escaped)
691 delimiter = self._D
693 # Form big string for easy check of delimiter clash
694 combined = "".join("".join(str(s) for s in k) for k in nameTuples)
696 # Try a delimiter and keep trying until we get something that
697 # works.
698 ntries = 0
699 while delimiter in combined:
700 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.")
701 ntries += 1
703 if ntries > 100:
704 raise ValueError(f"Unable to determine a delimiter for Config {self}")
706 # try another one
707 while True:
708 delimiter = chr(ord(delimiter)+1)
709 if not delimiter.isalnum():
710 break
712 log.debug(f"Using delimiter {delimiter!r}")
714 # Form the keys, escaping the delimiter if necessary
715 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k)
716 for k in nameTuples]
717 return strings
719 def asArray(self, name):
720 """Get a value as an array.
722 May contain one or more elements.
724 Parameters
725 ----------
726 name : `str`
727 Key to use to retrieve value.
729 Returns
730 -------
731 array : `collections.abc.Sequence`
732 The value corresponding to name, but guaranteed to be returned
733 as a list with at least one element. If the value is a
734 `~collections.abc.Sequence` (and not a `str`) the value itself
735 will be returned, else the value will be the first element.
736 """
737 val = self.get(name)
738 if isinstance(val, str):
739 val = [val]
740 elif not isinstance(val, collections.abc.Sequence):
741 val = [val]
742 return val
744 def __eq__(self, other):
745 if isinstance(other, Config):
746 other = other._data
747 return self._data == other
749 def __ne__(self, other):
750 if isinstance(other, Config):
751 other = other._data
752 return self._data != other
754 #######
755 # i/o #
757 def dump(self, output: Optional[IO] = None) -> Optional[str]:
758 """Writes the config to a yaml stream.
760 Parameters
761 ----------
762 output : `IO`, optional
763 The YAML stream to use for output. If `None` the YAML content
764 will be returned.
766 Returns
767 -------
768 yaml : `str` or `None`
769 If a stream was given the stream will be used and the return
770 value will be `None`. If the stream was `None` the YAML
771 serialization will be returned as a string.
772 """
773 return yaml.safe_dump(self._data, output, default_flow_style=False)
775 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
776 defaultFileName: str = "butler.yaml",
777 overwrite: bool = True) -> None:
778 """Writes the config to location pointed to by given URI.
780 Currently supports 's3' and 'file' URI schemes.
782 Parameters
783 ----------
784 uri: `str` or `ButlerURI`
785 URI of location where the Config will be written.
786 updateFile : bool, optional
787 If True and uri does not end on a filename with extension, will
788 append `defaultFileName` to the target uri. True by default.
789 defaultFileName : bool, optional
790 The file name that will be appended to target uri if updateFile is
791 True and uri does not end on a file with an extension.
792 overwrite : bool, optional
793 If True the configuration will be written even if it already
794 exists at that location.
795 """
796 # Make local copy of URI or create new one
797 uri = ButlerURI(uri)
799 if updateFile and not uri.getExtension():
800 uri.updateFile(defaultFileName)
802 uri.write(self.dump().encode(), overwrite=overwrite)
803 self.configFile = uri
805 @staticmethod
806 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True):
807 """Generic helper function for updating specific config parameters.
809 Allows for named parameters to be set to new values in bulk, and
810 for other values to be set by copying from a reference config.
812 Assumes that the supplied config is compatible with ``configType``
813 and will attach the updated values to the supplied config by
814 looking for the related component key. It is assumed that
815 ``config`` and ``full`` are from the same part of the
816 configuration hierarchy.
818 Parameters
819 ----------
820 configType : `ConfigSubset`
821 Config type to use to extract relevant items from ``config``.
822 config : `Config`
823 A `Config` to update. Only the subset understood by
824 the supplied `ConfigSubset` will be modified. Default values
825 will not be inserted and the content will not be validated
826 since mandatory keys are allowed to be missing until
827 populated later by merging.
828 full : `Config`
829 A complete config with all defaults expanded that can be
830 converted to a ``configType``. Read-only and will not be
831 modified by this method. Values are read from here if
832 ``toCopy`` is defined.
834 Repository-specific options that should not be obtained
835 from defaults when Butler instances are constructed
836 should be copied from ``full`` to ``config``.
837 toUpdate : `dict`, optional
838 A `dict` defining the keys to update and the new value to use.
839 The keys and values can be any supported by `Config`
840 assignment.
841 toCopy : `tuple`, optional
842 `tuple` of keys whose values should be copied from ``full``
843 into ``config``.
844 overwrite : `bool`, optional
845 If `False`, do not modify a value in ``config`` if the key
846 already exists. Default is always to overwrite.
848 Raises
849 ------
850 ValueError
851 Neither ``toUpdate`` not ``toCopy`` were defined.
852 """
853 if toUpdate is None and toCopy is None:
854 raise ValueError("One of toUpdate or toCopy parameters must be set.")
856 # If this is a parent configuration then we need to ensure that
857 # the supplied config has the relevant component key in it.
858 # If this is a parent configuration we add in the stub entry
859 # so that the ConfigSubset constructor will do the right thing.
860 # We check full for this since that is guaranteed to be complete.
861 if configType.component in full and configType.component not in config:
862 config[configType.component] = {}
864 # Extract the part of the config we wish to update
865 localConfig = configType(config, mergeDefaults=False, validate=False)
867 if toUpdate:
868 for key, value in toUpdate.items():
869 if key in localConfig and not overwrite:
870 log.debug("Not overriding key '%s' with value '%s' in config %s",
871 key, value, localConfig.__class__.__name__)
872 else:
873 localConfig[key] = value
875 if toCopy:
876 localFullConfig = configType(full, mergeDefaults=False)
877 for key in toCopy:
878 if key in localConfig and not overwrite:
879 log.debug("Not overriding key '%s' from defaults in config %s",
880 key, localConfig.__class__.__name__)
881 else:
882 localConfig[key] = localFullConfig[key]
884 # Reattach to parent if this is a child config
885 if configType.component in config:
886 config[configType.component] = localConfig
887 else:
888 config.update(localConfig)
890 def toDict(self):
891 """Convert a `Config` to a standalone hierarchical `dict`.
893 Returns
894 -------
895 d : `dict`
896 The standalone hierarchical `dict` with any `Config` classes
897 in the hierarchy converted to `dict`.
899 Notes
900 -----
901 This can be useful when passing a Config to some code that
902 expects native Python types.
903 """
904 output = copy.deepcopy(self._data)
905 for k, v in output.items():
906 if isinstance(v, Config):
907 v = v.toDict()
908 output[k] = v
909 return output
912class ConfigSubset(Config):
913 """Config representing a subset of a more general configuration.
915 Subclasses define their own component and when given a configuration
916 that includes that component, the resulting configuration only includes
917 the subset. For example, your config might contain ``dimensions`` if it's
918 part of a global config and that subset will be stored. If ``dimensions``
919 can not be found it is assumed that the entire contents of the
920 configuration should be used.
922 Default values are read from the environment or supplied search paths
923 using the default configuration file name specified in the subclass.
924 This allows a configuration class to be instantiated without any
925 additional arguments.
927 Additional validation can be specified to check for keys that are mandatory
928 in the configuration.
930 Parameters
931 ----------
932 other : `Config` or `str` or `dict`
933 Argument specifying the configuration information as understood
934 by `Config`
935 validate : `bool`, optional
936 If `True` required keys will be checked to ensure configuration
937 consistency.
938 mergeDefaults : `bool`, optional
939 If `True` defaults will be read and the supplied config will
940 be combined with the defaults, with the supplied valiues taking
941 precedence.
942 searchPaths : `list` or `tuple`, optional
943 Explicit additional paths to search for defaults. They should
944 be supplied in priority order. These paths have higher priority
945 than those read from the environment in
946 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
947 the local file system or URIs, `ButlerURI`.
948 """
950 component: ClassVar[Optional[str]] = None
951 """Component to use from supplied config. Can be None. If specified the
952 key is not required. Can be a full dot-separated path to a component.
953 """
955 requiredKeys: ClassVar[Sequence[str]] = ()
956 """Keys that are required to be specified in the configuration.
957 """
959 defaultConfigFile: ClassVar[Optional[str]] = None
960 """Name of the file containing defaults for this config class.
961 """
963 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None):
965 # Create a blank object to receive the defaults
966 # Once we have the defaults we then update with the external values
967 super().__init__()
969 # Create a standard Config rather than subset
970 externalConfig = Config(other)
972 # Select the part we need from it
973 # To simplify the use of !include we also check for the existence of
974 # component.component (since the included files can themselves
975 # include the component name)
976 if self.component is not None:
977 doubled = (self.component, self.component)
978 # Must check for double depth first
979 if doubled in externalConfig:
980 externalConfig = externalConfig[doubled]
981 elif self.component in externalConfig:
982 externalConfig._data = externalConfig._data[self.component]
984 # Default files read to create this configuration
985 self.filesRead = []
987 # Assume we are not looking up child configurations
988 containerKey = None
990 # Sometimes we do not want to merge with defaults.
991 if mergeDefaults:
993 # Supplied search paths have highest priority
994 fullSearchPath = []
995 if searchPaths:
996 fullSearchPath.extend(searchPaths)
998 # Read default paths from enviroment
999 fullSearchPath.extend(self.defaultSearchPaths())
1001 # There are two places to find defaults for this particular config
1002 # - The "defaultConfigFile" defined in the subclass
1003 # - The class specified in the "cls" element in the config.
1004 # Read cls after merging in case it changes.
1005 if self.defaultConfigFile is not None:
1006 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile)
1008 # Can have a class specification in the external config (priority)
1009 # or from the defaults.
1010 pytype = None
1011 if "cls" in externalConfig:
1012 pytype = externalConfig["cls"]
1013 elif "cls" in self:
1014 pytype = self["cls"]
1016 if pytype is not None:
1017 try:
1018 cls = doImport(pytype)
1019 except ImportError as e:
1020 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e
1021 defaultsFile = cls.defaultConfigFile
1022 if defaultsFile is not None:
1023 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile)
1025 # Get the container key in case we need it
1026 try:
1027 containerKey = cls.containerKey
1028 except AttributeError:
1029 pass
1031 # Now update this object with the external values so that the external
1032 # values always override the defaults
1033 self.update(externalConfig)
1035 # If this configuration has child configurations of the same
1036 # config class, we need to expand those defaults as well.
1038 if mergeDefaults and containerKey is not None and containerKey in self:
1039 for idx, subConfig in enumerate(self[containerKey]):
1040 self[containerKey, idx] = type(self)(other=subConfig, validate=validate,
1041 mergeDefaults=mergeDefaults,
1042 searchPaths=searchPaths)
1044 if validate:
1045 self.validate()
1047 @classmethod
1048 def defaultSearchPaths(cls):
1049 """Read the environment to determine search paths to use for global
1050 defaults.
1052 Global defaults, at lowest priority, are found in the ``config``
1053 directory of the butler source tree. Additional defaults can be
1054 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS``
1055 which is a PATH-like variable where paths at the front of the list
1056 have priority over those later.
1058 Returns
1059 -------
1060 paths : `list`
1061 Returns a list of paths to search. The returned order is in
1062 priority with the highest priority paths first. The butler config
1063 configuration resources will not be included here but will
1064 always be searched last.
1066 Notes
1067 -----
1068 The environment variable is split on the standard ``:`` path separator.
1069 This currently makes it incompatible with usage of URIs.
1070 """
1071 # We can pick up defaults from multiple search paths
1072 # We fill defaults by using the butler config path and then
1073 # the config path environment variable in reverse order.
1074 defaultsPaths = []
1076 if CONFIG_PATH in os.environ:
1077 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep)
1078 defaultsPaths.extend(externalPaths)
1080 # Add the package defaults as a resource
1081 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs",
1082 forceDirectory=True))
1083 return defaultsPaths
1085 def _updateWithConfigsFromPath(self, searchPaths, configFile):
1086 """Search the supplied paths, merging the configuration values
1088 The values read will override values currently stored in the object.
1089 Every file found in the path will be read, such that the earlier
1090 path entries have higher priority.
1092 Parameters
1093 ----------
1094 searchPaths : `list` of `ButlerURI`, `str`
1095 Paths to search for the supplied configFile. This path
1096 is the priority order, such that files read from the
1097 first path entry will be selected over those read from
1098 a later path. Can contain `str` referring to the local file
1099 system or a URI string.
1100 configFile : `ButlerURI`
1101 File to locate in path. If absolute path it will be read
1102 directly and the search path will not be used. Can be a URI
1103 to an explicit resource (which will ignore the search path)
1104 which is assumed to exist.
1105 """
1106 uri = ButlerURI(configFile)
1107 if uri.isabs() and uri.exists():
1108 # Assume this resource exists
1109 self._updateWithOtherConfigFile(configFile)
1110 self.filesRead.append(configFile)
1111 else:
1112 # Reverse order so that high priority entries
1113 # update the object last.
1114 for pathDir in reversed(searchPaths):
1115 if isinstance(pathDir, (str, ButlerURI)):
1116 pathDir = ButlerURI(pathDir, forceDirectory=True)
1117 file = pathDir.join(configFile)
1118 if file.exists():
1119 self.filesRead.append(file)
1120 self._updateWithOtherConfigFile(file)
1121 else:
1122 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}")
1124 def _updateWithOtherConfigFile(self, file):
1125 """Read in some defaults and update.
1127 Update the configuration by reading the supplied file as a config
1128 of this class, and merging such that these values override the
1129 current values. Contents of the external config are not validated.
1131 Parameters
1132 ----------
1133 file : `Config`, `str`, `ButlerURI`, or `dict`
1134 Entity that can be converted to a `ConfigSubset`.
1135 """
1136 # Use this class to read the defaults so that subsetting can happen
1137 # correctly.
1138 externalConfig = type(self)(file, validate=False, mergeDefaults=False)
1139 self.update(externalConfig)
1141 def validate(self):
1142 """Check that mandatory keys are present in this configuration.
1144 Ignored if ``requiredKeys`` is empty."""
1145 # Validation
1146 missing = [k for k in self.requiredKeys if k not in self._data]
1147 if missing:
1148 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")