Coverage for python/lsst/daf/butler/core/config.py : 42%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Configuration control."""
26__all__ = ("Config", "ConfigSubset")
28import collections
29import copy
30import json
31import logging
32import pprint
33import os
34import yaml
35import sys
36from pathlib import Path
37from yaml.representer import Representer
38import io
39from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union
41from lsst.utils import doImport
42from ._butlerUri import ButlerURI
44yaml.add_representer(collections.defaultdict, Representer.represent_dict)
47# Config module logger
48log = logging.getLogger(__name__)
50# PATH-like environment variable to use for defaults.
51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH"
53try:
54 yamlLoader = yaml.CSafeLoader
55except AttributeError:
56 # Not all installations have the C library
57 # (but assume for mypy's sake that they're the same)
58 yamlLoader = yaml.SafeLoader # type: ignore
61def _doUpdate(d, u):
62 if not isinstance(u, collections.abc.Mapping) or \ 62 ↛ 64line 62 didn't jump to line 64, because the condition on line 62 was never true
63 not isinstance(d, collections.abc.MutableMapping):
64 raise RuntimeError("Only call update with Mapping, not {}".format(type(d)))
65 for k, v in u.items():
66 if isinstance(v, collections.abc.Mapping):
67 d[k] = _doUpdate(d.get(k, {}), v)
68 else:
69 d[k] = v
70 return d
73class Loader(yamlLoader):
74 """YAML Loader that supports file include directives.
76 Uses ``!include`` directive in a YAML file to point to another
77 YAML file to be included. The path in the include directive is relative
78 to the file containing that directive.
80 storageClasses: !include storageClasses.yaml
82 Examples
83 --------
84 >>> with open("document.yaml", "r") as f:
85 data = yaml.load(f, Loader=Loader)
87 Notes
88 -----
89 See https://davidchall.github.io/yaml-includes.html
90 """
92 def __init__(self, stream):
93 super().__init__(stream)
94 # if this is a string and not a stream we may well lack a name
95 try:
96 self._root = ButlerURI(stream.name)
97 except AttributeError:
98 # No choice but to assume a local filesystem
99 self._root = ButlerURI("no-file.yaml")
100 Loader.add_constructor("!include", Loader.include)
102 def include(self, node):
103 result: Union[List[Any], Dict[str, Any]]
104 if isinstance(node, yaml.ScalarNode):
105 return self.extractFile(self.construct_scalar(node))
107 elif isinstance(node, yaml.SequenceNode):
108 result = []
109 for filename in self.construct_sequence(node):
110 result.append(self.extractFile(filename))
111 return result
113 elif isinstance(node, yaml.MappingNode):
114 result = {}
115 for k, v in self.construct_mapping(node).items():
116 result[k] = self.extractFile(v)
117 return result
119 else:
120 print("Error:: unrecognised node type in !include statement", file=sys.stderr)
121 raise yaml.constructor.ConstructorError
123 def extractFile(self, filename):
124 # It is possible for the !include to point to an explicit URI
125 # instead of a relative URI, therefore we first see if it is
126 # scheme-less or not. If it has a scheme we use it directly
127 # if it is scheme-less we use it relative to the file root.
128 requesteduri = ButlerURI(filename, forceAbsolute=False)
130 if requesteduri.scheme:
131 fileuri = requesteduri
132 else:
133 fileuri = self._root.updatedFile(filename)
135 log.debug("Opening YAML file via !include: %s", fileuri)
137 # Read all the data from the resource
138 data = fileuri.read()
140 # Store the bytes into a BytesIO so we can attach a .name
141 stream = io.BytesIO(data)
142 stream.name = fileuri.geturl()
143 return yaml.load(stream, Loader)
146class Config(collections.abc.MutableMapping):
147 r"""Implements a datatype that is used by `Butler` for configuration.
149 It is essentially a `dict` with key/value pairs, including nested dicts
150 (as values). In fact, it can be initialized with a `dict`.
151 This is explained next:
153 Config extends the `dict` api so that hierarchical values may be accessed
154 with delimited notation or as a tuple. If a string is given the delimiter
155 is picked up from the first character in that string. For example,
156 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``,
157 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome.
158 If the first character is alphanumeric, no delimiter will be used.
159 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``.
160 Unicode characters can be used as the delimiter for distinctiveness if
161 required.
163 If a key in the hierarchy starts with a non-alphanumeric character care
164 should be used to ensure that either the tuple interface is used or
165 a distinct delimiter is always given in string form.
167 Finally, the delimiter can be escaped if it is part of a key and also
168 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in
169 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is
170 always better to use a different delimiter in these cases.
172 Note that adding a multi-level key implicitly creates any nesting levels
173 that do not exist, but removing multi-level keys does not automatically
174 remove empty nesting levels. As a result:
176 >>> c = Config()
177 >>> c[".a.b"] = 1
178 >>> del c[".a.b"]
179 >>> c["a"]
180 Config({'a': {}})
182 Storage formats supported:
184 - yaml: read and write is supported.
185 - json: read and write is supported but no ``!include`` directive.
187 Parameters
188 ----------
189 other : `str` or `Config` or `dict` or `ButlerURI` or `pathlib.Path`
190 Other source of configuration, can be:
192 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end
193 with ".yaml".
194 - (`Config`) Copies the other Config's values into this one.
195 - (`dict`) Copies the values from the dict into this Config.
197 If `None` is provided an empty `Config` will be created.
198 """
200 _D: str = "→"
201 """Default internal delimiter to use for components in the hierarchy when
202 constructing keys for external use (see `Config.names()`)."""
204 includeKey: ClassVar[str] = "includeConfigs"
205 """Key used to indicate that another config should be included at this
206 part of the hierarchy."""
208 resourcesPackage: str = "lsst.daf.butler"
209 """Package to search for default configuration data. The resources
210 themselves will be within a ``configs`` resource hierarchy."""
212 def __init__(self, other=None):
213 self._data: Dict[str, Any] = {}
214 self.configFile = None
216 if other is None:
217 return
219 if isinstance(other, Config):
220 self._data = copy.deepcopy(other._data)
221 self.configFile = other.configFile
222 elif isinstance(other, collections.abc.Mapping):
223 self.update(other)
224 elif isinstance(other, (str, ButlerURI, Path)): 224 ↛ 231line 224 didn't jump to line 231, because the condition on line 224 was never false
225 # if other is a string, assume it is a file path/URI
226 self.__initFromUri(other)
227 self._processExplicitIncludes()
228 else:
229 # if the config specified by other could not be recognized raise
230 # a runtime error.
231 raise RuntimeError(f"A Config could not be loaded from other: {other}")
233 def ppprint(self):
234 """Return config as formatted readable string.
236 Examples
237 --------
238 use: ``pdb> print(myConfigObject.ppprint())``
240 Returns
241 -------
242 s : `str`
243 A prettyprint formatted string representing the config
244 """
245 return pprint.pformat(self._data, indent=2, width=1)
247 def __repr__(self):
248 return f"{type(self).__name__}({self._data!r})"
250 def __str__(self):
251 return self.ppprint()
253 def __len__(self):
254 return len(self._data)
256 def __iter__(self):
257 return iter(self._data)
259 def copy(self):
260 return type(self)(self)
262 @classmethod
263 def fromString(cls, string: str, format: str = "yaml") -> Config:
264 """Create a new Config instance from a serialized string.
266 Parameters
267 ----------
268 string : `str`
269 String containing content in specified format
270 format : `str`, optional
271 Format of the supplied string. Can be ``json`` or ``yaml``.
273 Returns
274 -------
275 c : `Config`
276 Newly-constructed Config.
277 """
278 if format == "yaml":
279 new_config = cls().__initFromYaml(string)
280 elif format == "json":
281 new_config = cls().__initFromJson(string)
282 else:
283 raise ValueError(f"Unexpected format of string: {format}")
284 new_config._processExplicitIncludes()
285 return new_config
287 @classmethod
288 def fromYaml(cls, string: str) -> Config:
289 """Create a new Config instance from a YAML string.
291 Parameters
292 ----------
293 string : `str`
294 String containing content in YAML format
296 Returns
297 -------
298 c : `Config`
299 Newly-constructed Config.
300 """
301 return cls.fromString(string, format="yaml")
303 def __initFromUri(self, path: Union[str, ButlerURI, Path]) -> None:
304 """Load a file from a path or an URI.
306 Parameters
307 ----------
308 path : `str`
309 Path or a URI to a persisted config file.
310 """
311 uri = ButlerURI(path)
312 ext = uri.getExtension()
313 if ext == ".yaml": 313 ↛ 320line 313 didn't jump to line 320, because the condition on line 313 was never false
314 log.debug("Opening YAML config file: %s", uri.geturl())
315 content = uri.read()
316 # Use a stream so we can name it
317 stream = io.BytesIO(content)
318 stream.name = uri.geturl()
319 self.__initFromYaml(stream)
320 elif ext == ".json":
321 log.debug("Opening JSON config file: %s", uri.geturl())
322 content = uri.read()
323 self.__initFromJson(content)
324 else:
325 # This URI does not have a valid extension. It might be because
326 # we ended up with a directory and not a file. Before we complain
327 # about an extension, do an existence check. No need to do
328 # the (possibly expensive) existence check in the default code
329 # path above because we will find out soon enough that the file
330 # is not there.
331 if not uri.exists():
332 raise FileNotFoundError(f"Config location {uri} does not exist.")
333 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}")
334 self.configFile = uri
336 def __initFromYaml(self, stream):
337 """Load a YAML config from any readable stream that contains one.
339 Parameters
340 ----------
341 stream: `IO` or `str`
342 Stream to pass to the YAML loader. Accepts anything that
343 `yaml.load` accepts. This can include a string as well as an
344 IO stream.
346 Raises
347 ------
348 yaml.YAMLError
349 If there is an error loading the file.
350 """
351 content = yaml.load(stream, Loader=Loader)
352 if content is None: 352 ↛ 353line 352 didn't jump to line 353, because the condition on line 352 was never true
353 content = {}
354 self._data = content
355 return self
357 def __initFromJson(self, stream):
358 """Load a JSON config from any readable stream that contains one.
360 Parameters
361 ----------
362 stream: `IO` or `str`
363 Stream to pass to the JSON loader. This can include a string as
364 well as an IO stream.
366 Raises
367 ------
368 TypeError:
369 Raised if there is an error loading the content.
370 """
371 if isinstance(stream, (bytes, str)):
372 content = json.loads(stream)
373 else:
374 content = json.load(stream)
375 if content is None:
376 content = {}
377 self._data = content
378 return self
380 def _processExplicitIncludes(self):
381 """Scan through the configuration searching for the special includes.
383 Looks for ``includeConfigs`` directive and processes the includes.
384 """
385 # Search paths for config files
386 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)]
387 if self.configFile is not None: 387 ↛ 395line 387 didn't jump to line 395, because the condition on line 387 was never false
388 if isinstance(self.configFile, ButlerURI): 388 ↛ 391line 388 didn't jump to line 391, because the condition on line 388 was never false
389 configDir = self.configFile.dirname()
390 else:
391 raise RuntimeError(f"Unexpected type for config file: {self.configFile}")
392 searchPaths.append(configDir)
394 # Ensure we know what delimiter to use
395 names = self.nameTuples()
396 for path in names:
397 if path[-1] == self.includeKey: 397 ↛ 399line 397 didn't jump to line 399, because the condition on line 397 was never true
399 log.debug("Processing file include directive at %s", self._D + self._D.join(path))
400 basePath = path[:-1]
402 # Extract the includes and then delete them from the config
403 includes = self[path]
404 del self[path]
406 # Be consistent and convert to a list
407 if not isinstance(includes, list):
408 includes = [includes]
410 # Read each file assuming it is a reference to a file
411 # The file can be relative to config file or cwd
412 # ConfigSubset search paths are not used
413 subConfigs = []
414 for fileName in includes:
415 # Expand any shell variables -- this could be URI
416 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False)
417 found = None
418 if fileName.isabs():
419 found = fileName
420 else:
421 for dir in searchPaths:
422 if isinstance(dir, ButlerURI):
423 specific = dir.join(fileName.path)
424 # Remote resource check might be expensive
425 if specific.exists():
426 found = specific
427 else:
428 log.warning("Do not understand search path entry '%s' of type %s",
429 dir, type(dir).__name__)
430 if not found:
431 raise RuntimeError(f"Unable to find referenced include file: {fileName}")
433 # Read the referenced Config as a Config
434 subConfigs.append(type(self)(found))
436 # Now we need to merge these sub configs with the current
437 # information that was present in this node in the config
438 # tree with precedence given to the explicit values
439 newConfig = subConfigs.pop(0)
440 for sc in subConfigs:
441 newConfig.update(sc)
443 # Explicit values take precedence
444 if not basePath:
445 # This is an include at the root config
446 newConfig.update(self)
447 # Replace the current config
448 self._data = newConfig._data
449 else:
450 newConfig.update(self[basePath])
451 # And reattach to the base config
452 self[basePath] = newConfig
454 @staticmethod
455 def _splitIntoKeys(key):
456 r"""Split the argument for get/set/in into a hierarchical list.
458 Parameters
459 ----------
460 key : `str` or iterable
461 Argument given to get/set/in. If an iterable is provided it will
462 be converted to a list. If the first character of the string
463 is not an alphanumeric character then it will be used as the
464 delimiter for the purposes of splitting the remainder of the
465 string. If the delimiter is also in one of the keys then it
466 can be escaped using ``\``. There is no default delimiter.
468 Returns
469 -------
470 keys : `list`
471 Hierarchical keys as a `list`.
472 """
473 if isinstance(key, str):
474 if not key[0].isalnum(): 474 ↛ 475line 474 didn't jump to line 475, because the condition on line 474 was never true
475 d = key[0]
476 key = key[1:]
477 else:
478 return [key, ]
479 escaped = f"\\{d}"
480 temp = None
481 if escaped in key:
482 # Complain at the attempt to escape the escape
483 doubled = fr"\{escaped}"
484 if doubled in key:
485 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})"
486 " is not yet supported.")
487 # Replace with a character that won't be in the string
488 temp = "\r"
489 if temp in key or d == temp:
490 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as"
491 " delimiter if escaping the delimiter")
492 key = key.replace(escaped, temp)
493 hierarchy = key.split(d)
494 if temp:
495 hierarchy = [h.replace(temp, d) for h in hierarchy]
496 return hierarchy
497 elif isinstance(key, collections.abc.Iterable): 497 ↛ 501line 497 didn't jump to line 501, because the condition on line 497 was never false
498 return list(key)
499 else:
500 # Not sure what this is so try it anyway
501 return [key, ]
503 def _getKeyHierarchy(self, name):
504 """Retrieve the key hierarchy for accessing the Config.
506 Parameters
507 ----------
508 name : `str` or `tuple`
509 Delimited string or `tuple` of hierarchical keys.
511 Returns
512 -------
513 hierarchy : `list` of `str`
514 Hierarchy to use as a `list`. If the name is available directly
515 as a key in the Config it will be used regardless of the presence
516 of any nominal delimiter.
517 """
518 if name in self._data:
519 keys = [name, ]
520 else:
521 keys = self._splitIntoKeys(name)
522 return keys
524 def _findInHierarchy(self, keys, create=False):
525 """Look for hierarchy of keys in Config.
527 Parameters
528 ----------
529 keys : `list` or `tuple`
530 Keys to search in hierarchy.
531 create : `bool`, optional
532 If `True`, if a part of the hierarchy does not exist, insert an
533 empty `dict` into the hierarchy.
535 Returns
536 -------
537 hierarchy : `list`
538 List of the value corresponding to each key in the supplied
539 hierarchy. Only keys that exist in the hierarchy will have
540 a value.
541 complete : `bool`
542 `True` if the full hierarchy exists and the final element
543 in ``hierarchy`` is the value of relevant value.
544 """
545 d = self._data
547 def checkNextItem(k, d, create):
548 """See if k is in d and if it is return the new child."""
549 nextVal = None
550 isThere = False
551 if d is None: 551 ↛ 553line 551 didn't jump to line 553, because the condition on line 551 was never true
552 # We have gone past the end of the hierarchy
553 pass
554 elif isinstance(d, collections.abc.Sequence): 554 ↛ 559line 554 didn't jump to line 559, because the condition on line 554 was never true
555 # Check sequence first because for lists
556 # __contains__ checks whether value is found in list
557 # not whether the index exists in list. When we traverse
558 # the hierarchy we are interested in the index.
559 try:
560 nextVal = d[int(k)]
561 isThere = True
562 except IndexError:
563 pass
564 except ValueError:
565 isThere = k in d
566 elif k in d:
567 nextVal = d[k]
568 isThere = True
569 elif create: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true
570 d[k] = {}
571 nextVal = d[k]
572 isThere = True
573 return nextVal, isThere
575 hierarchy = []
576 complete = True
577 for k in keys:
578 d, isThere = checkNextItem(k, d, create)
579 if isThere:
580 hierarchy.append(d)
581 else:
582 complete = False
583 break
585 return hierarchy, complete
587 def __getitem__(self, name):
588 # Override the split for the simple case where there is an exact
589 # match. This allows `Config.items()` to work via a simple
590 # __iter__ implementation that returns top level keys of
591 # self._data.
592 keys = self._getKeyHierarchy(name)
594 hierarchy, complete = self._findInHierarchy(keys)
595 if not complete: 595 ↛ 596line 595 didn't jump to line 596, because the condition on line 595 was never true
596 raise KeyError(f"{name} not found")
597 data = hierarchy[-1]
599 if isinstance(data, collections.abc.Mapping):
600 data = Config(data)
601 # Ensure that child configs inherit the parent internal delimiter
602 if self._D != Config._D: 602 ↛ 603line 602 didn't jump to line 603, because the condition on line 602 was never true
603 data._D = self._D
604 return data
606 def __setitem__(self, name, value):
607 keys = self._getKeyHierarchy(name)
608 last = keys.pop()
609 if isinstance(value, Config):
610 value = copy.deepcopy(value._data)
612 hierarchy, complete = self._findInHierarchy(keys, create=True)
613 if hierarchy:
614 data = hierarchy[-1]
615 else:
616 data = self._data
618 try:
619 data[last] = value
620 except TypeError:
621 data[int(last)] = value
623 def __contains__(self, key):
624 keys = self._getKeyHierarchy(key)
625 hierarchy, complete = self._findInHierarchy(keys)
626 return complete
628 def __delitem__(self, key):
629 keys = self._getKeyHierarchy(key)
630 last = keys.pop()
631 hierarchy, complete = self._findInHierarchy(keys)
632 if complete: 632 ↛ 639line 632 didn't jump to line 639, because the condition on line 632 was never false
633 if hierarchy: 633 ↛ 634line 633 didn't jump to line 634, because the condition on line 633 was never true
634 data = hierarchy[-1]
635 else:
636 data = self._data
637 del data[last]
638 else:
639 raise KeyError(f"{key} not found in Config")
641 def update(self, other):
642 """Update config from other `Config` or `dict`.
644 Like `dict.update()`, but will add or modify keys in nested dicts,
645 instead of overwriting the nested dict entirely.
647 Parameters
648 ----------
649 other : `dict` or `Config`
650 Source of configuration:
652 Examples
653 --------
654 >>> c = Config({"a": {"b": 1}})
655 >>> c.update({"a": {"c": 2}})
656 >>> print(c)
657 {'a': {'b': 1, 'c': 2}}
659 >>> foo = {"a": {"b": 1}}
660 >>> foo.update({"a": {"c": 2}})
661 >>> print(foo)
662 {'a': {'c': 2}}
663 """
664 _doUpdate(self._data, other)
666 def merge(self, other):
667 """Merge another Config into this one.
669 Like `Config.update()`, but will add keys & values from other that
670 DO NOT EXIST in self.
672 Keys and values that already exist in self will NOT be overwritten.
674 Parameters
675 ----------
676 other : `dict` or `Config`
677 Source of configuration:
678 """
679 if not isinstance(other, collections.abc.Mapping):
680 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}")
682 # Convert the supplied mapping to a Config for consistency
683 # This will do a deepcopy if it is already a Config
684 otherCopy = Config(other)
685 otherCopy.update(self)
686 self._data = otherCopy._data
688 def nameTuples(self, topLevelOnly=False):
689 """Get tuples representing the name hierarchies of all keys.
691 The tuples returned from this method are guaranteed to be usable
692 to access items in the configuration object.
694 Parameters
695 ----------
696 topLevelOnly : `bool`, optional
697 If False, the default, a full hierarchy of names is returned.
698 If True, only the top level are returned.
700 Returns
701 -------
702 names : `list` of `tuple` of `str`
703 List of all names present in the `Config` where each element
704 in the list is a `tuple` of strings representing the hierarchy.
705 """
706 if topLevelOnly: 706 ↛ 707line 706 didn't jump to line 707, because the condition on line 706 was never true
707 return list((k,) for k in self)
709 def getKeysAsTuples(d, keys, base):
710 if isinstance(d, collections.abc.Sequence):
711 theseKeys = range(len(d))
712 else:
713 theseKeys = d.keys()
714 for key in theseKeys:
715 val = d[key]
716 levelKey = base + (key,) if base is not None else (key,)
717 keys.append(levelKey)
718 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \
719 and not isinstance(val, str):
720 getKeysAsTuples(val, keys, levelKey)
721 keys: List[Tuple[str, ...]] = []
722 getKeysAsTuples(self._data, keys, None)
723 return keys
725 def names(self, topLevelOnly=False, delimiter=None):
726 """Get a delimited name of all the keys in the hierarchy.
728 The values returned from this method are guaranteed to be usable
729 to access items in the configuration object.
731 Parameters
732 ----------
733 topLevelOnly : `bool`, optional
734 If False, the default, a full hierarchy of names is returned.
735 If True, only the top level are returned.
736 delimiter : `str`, optional
737 Delimiter to use when forming the keys. If the delimiter is
738 present in any of the keys, it will be escaped in the returned
739 names. If `None` given a delimiter will be automatically provided.
740 The delimiter can not be alphanumeric.
742 Returns
743 -------
744 names : `list` of `str`
745 List of all names present in the `Config`.
747 Notes
748 -----
749 This is different than the built-in method `dict.keys`, which will
750 return only the first level keys.
752 Raises
753 ------
754 ValueError:
755 The supplied delimiter is alphanumeric.
756 """
757 if topLevelOnly:
758 return list(self.keys())
760 # Get all the tuples of hierarchical keys
761 nameTuples = self.nameTuples()
763 if delimiter is not None and delimiter.isalnum():
764 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.")
766 if delimiter is None:
767 # Start with something, and ensure it does not need to be
768 # escaped (it is much easier to understand if not escaped)
769 delimiter = self._D
771 # Form big string for easy check of delimiter clash
772 combined = "".join("".join(str(s) for s in k) for k in nameTuples)
774 # Try a delimiter and keep trying until we get something that
775 # works.
776 ntries = 0
777 while delimiter in combined:
778 log.debug("Delimiter '%s' could not be used. Trying another.", delimiter)
779 ntries += 1
781 if ntries > 100:
782 raise ValueError(f"Unable to determine a delimiter for Config {self}")
784 # try another one
785 while True:
786 delimiter = chr(ord(delimiter)+1)
787 if not delimiter.isalnum():
788 break
790 log.debug("Using delimiter %r", delimiter)
792 # Form the keys, escaping the delimiter if necessary
793 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k)
794 for k in nameTuples]
795 return strings
797 def asArray(self, name):
798 """Get a value as an array.
800 May contain one or more elements.
802 Parameters
803 ----------
804 name : `str`
805 Key to use to retrieve value.
807 Returns
808 -------
809 array : `collections.abc.Sequence`
810 The value corresponding to name, but guaranteed to be returned
811 as a list with at least one element. If the value is a
812 `~collections.abc.Sequence` (and not a `str`) the value itself
813 will be returned, else the value will be the first element.
814 """
815 val = self.get(name)
816 if isinstance(val, str):
817 val = [val]
818 elif not isinstance(val, collections.abc.Sequence):
819 val = [val]
820 return val
822 def __eq__(self, other):
823 if isinstance(other, Config):
824 other = other._data
825 return self._data == other
827 def __ne__(self, other):
828 if isinstance(other, Config):
829 other = other._data
830 return self._data != other
832 #######
833 # i/o #
835 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]:
836 """Write the config to an output stream.
838 Parameters
839 ----------
840 output : `IO`, optional
841 The stream to use for output. If `None` the serialized content
842 will be returned.
843 format : `str`, optional
844 The format to use for the output. Can be "yaml" or "json".
846 Returns
847 -------
848 serialized : `str` or `None`
849 If a stream was given the stream will be used and the return
850 value will be `None`. If the stream was `None` the
851 serialization will be returned as a string.
852 """
853 if format == "yaml":
854 return yaml.safe_dump(self._data, output, default_flow_style=False)
855 elif format == "json":
856 if output is not None:
857 json.dump(self._data, output, ensure_ascii=False)
858 return None
859 else:
860 return json.dumps(self._data, ensure_ascii=False)
861 raise ValueError(f"Unsupported format for Config serialization: {format}")
863 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
864 defaultFileName: str = "butler.yaml",
865 overwrite: bool = True) -> None:
866 """Write the config to location pointed to by given URI.
868 Currently supports 's3' and 'file' URI schemes.
870 Parameters
871 ----------
872 uri: `str` or `ButlerURI`
873 URI of location where the Config will be written.
874 updateFile : bool, optional
875 If True and uri does not end on a filename with extension, will
876 append `defaultFileName` to the target uri. True by default.
877 defaultFileName : bool, optional
878 The file name that will be appended to target uri if updateFile is
879 True and uri does not end on a file with an extension.
880 overwrite : bool, optional
881 If True the configuration will be written even if it already
882 exists at that location.
883 """
884 # Make local copy of URI or create new one
885 uri = ButlerURI(uri)
887 if updateFile and not uri.getExtension():
888 uri = uri.updatedFile(defaultFileName)
890 # Try to work out the format from the extension
891 ext = uri.getExtension()
892 format = ext[1:].lower()
894 output = self.dump(format=format)
895 assert output is not None, "Config.dump guarantees not-None return when output arg is None"
896 uri.write(output.encode(), overwrite=overwrite)
897 self.configFile = uri
899 @staticmethod
900 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True, toMerge=None):
901 """Update specific config parameters.
903 Allows for named parameters to be set to new values in bulk, and
904 for other values to be set by copying from a reference config.
906 Assumes that the supplied config is compatible with ``configType``
907 and will attach the updated values to the supplied config by
908 looking for the related component key. It is assumed that
909 ``config`` and ``full`` are from the same part of the
910 configuration hierarchy.
912 Parameters
913 ----------
914 configType : `ConfigSubset`
915 Config type to use to extract relevant items from ``config``.
916 config : `Config`
917 A `Config` to update. Only the subset understood by
918 the supplied `ConfigSubset` will be modified. Default values
919 will not be inserted and the content will not be validated
920 since mandatory keys are allowed to be missing until
921 populated later by merging.
922 full : `Config`
923 A complete config with all defaults expanded that can be
924 converted to a ``configType``. Read-only and will not be
925 modified by this method. Values are read from here if
926 ``toCopy`` is defined.
928 Repository-specific options that should not be obtained
929 from defaults when Butler instances are constructed
930 should be copied from ``full`` to ``config``.
931 toUpdate : `dict`, optional
932 A `dict` defining the keys to update and the new value to use.
933 The keys and values can be any supported by `Config`
934 assignment.
935 toCopy : `tuple`, optional
936 `tuple` of keys whose values should be copied from ``full``
937 into ``config``.
938 overwrite : `bool`, optional
939 If `False`, do not modify a value in ``config`` if the key
940 already exists. Default is always to overwrite.
941 toMerge : `tuple`, optional
942 Keys to merge content from full to config without overwriting
943 pre-existing values. Only works if the key refers to a hierarchy.
944 The ``overwrite`` flag is ignored.
946 Raises
947 ------
948 ValueError
949 Neither ``toUpdate``, ``toCopy`` nor ``toMerge`` were defined.
950 """
951 if toUpdate is None and toCopy is None and toMerge is None:
952 raise ValueError("At least one of toUpdate, toCopy, or toMerge parameters must be set.")
954 # If this is a parent configuration then we need to ensure that
955 # the supplied config has the relevant component key in it.
956 # If this is a parent configuration we add in the stub entry
957 # so that the ConfigSubset constructor will do the right thing.
958 # We check full for this since that is guaranteed to be complete.
959 if configType.component in full and configType.component not in config:
960 config[configType.component] = {}
962 # Extract the part of the config we wish to update
963 localConfig = configType(config, mergeDefaults=False, validate=False)
965 if toUpdate:
966 for key, value in toUpdate.items():
967 if key in localConfig and not overwrite:
968 log.debug("Not overriding key '%s' with value '%s' in config %s",
969 key, value, localConfig.__class__.__name__)
970 else:
971 localConfig[key] = value
973 if toCopy or toMerge:
974 localFullConfig = configType(full, mergeDefaults=False)
976 if toCopy:
977 for key in toCopy:
978 if key in localConfig and not overwrite:
979 log.debug("Not overriding key '%s' from defaults in config %s",
980 key, localConfig.__class__.__name__)
981 else:
982 localConfig[key] = localFullConfig[key]
983 if toMerge:
984 for key in toMerge:
985 if key in localConfig:
986 # Get the node from the config to do the merge
987 # but then have to reattach to the config.
988 subset = localConfig[key]
989 subset.merge(localFullConfig[key])
990 localConfig[key] = subset
991 else:
992 localConfig[key] = localFullConfig[key]
994 # Reattach to parent if this is a child config
995 if configType.component in config:
996 config[configType.component] = localConfig
997 else:
998 config.update(localConfig)
1000 def toDict(self):
1001 """Convert a `Config` to a standalone hierarchical `dict`.
1003 Returns
1004 -------
1005 d : `dict`
1006 The standalone hierarchical `dict` with any `Config` classes
1007 in the hierarchy converted to `dict`.
1009 Notes
1010 -----
1011 This can be useful when passing a Config to some code that
1012 expects native Python types.
1013 """
1014 output = copy.deepcopy(self._data)
1015 for k, v in output.items():
1016 if isinstance(v, Config):
1017 v = v.toDict()
1018 output[k] = v
1019 return output
1022class ConfigSubset(Config):
1023 """Config representing a subset of a more general configuration.
1025 Subclasses define their own component and when given a configuration
1026 that includes that component, the resulting configuration only includes
1027 the subset. For example, your config might contain ``dimensions`` if it's
1028 part of a global config and that subset will be stored. If ``dimensions``
1029 can not be found it is assumed that the entire contents of the
1030 configuration should be used.
1032 Default values are read from the environment or supplied search paths
1033 using the default configuration file name specified in the subclass.
1034 This allows a configuration class to be instantiated without any
1035 additional arguments.
1037 Additional validation can be specified to check for keys that are mandatory
1038 in the configuration.
1040 Parameters
1041 ----------
1042 other : `Config` or `str` or `dict`
1043 Argument specifying the configuration information as understood
1044 by `Config`
1045 validate : `bool`, optional
1046 If `True` required keys will be checked to ensure configuration
1047 consistency.
1048 mergeDefaults : `bool`, optional
1049 If `True` defaults will be read and the supplied config will
1050 be combined with the defaults, with the supplied valiues taking
1051 precedence.
1052 searchPaths : `list` or `tuple`, optional
1053 Explicit additional paths to search for defaults. They should
1054 be supplied in priority order. These paths have higher priority
1055 than those read from the environment in
1056 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
1057 the local file system or URIs, `ButlerURI`.
1058 """
1060 component: ClassVar[Optional[str]] = None
1061 """Component to use from supplied config. Can be None. If specified the
1062 key is not required. Can be a full dot-separated path to a component.
1063 """
1065 requiredKeys: ClassVar[Sequence[str]] = ()
1066 """Keys that are required to be specified in the configuration.
1067 """
1069 defaultConfigFile: ClassVar[Optional[str]] = None
1070 """Name of the file containing defaults for this config class.
1071 """
1073 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None):
1075 # Create a blank object to receive the defaults
1076 # Once we have the defaults we then update with the external values
1077 super().__init__()
1079 # Create a standard Config rather than subset
1080 externalConfig = Config(other)
1082 # Select the part we need from it
1083 # To simplify the use of !include we also check for the existence of
1084 # component.component (since the included files can themselves
1085 # include the component name)
1086 if self.component is not None: 1086 ↛ 1095line 1086 didn't jump to line 1095, because the condition on line 1086 was never false
1087 doubled = (self.component, self.component)
1088 # Must check for double depth first
1089 if doubled in externalConfig: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true
1090 externalConfig = externalConfig[doubled]
1091 elif self.component in externalConfig:
1092 externalConfig._data = externalConfig._data[self.component]
1094 # Default files read to create this configuration
1095 self.filesRead = []
1097 # Assume we are not looking up child configurations
1098 containerKey = None
1100 # Sometimes we do not want to merge with defaults.
1101 if mergeDefaults:
1103 # Supplied search paths have highest priority
1104 fullSearchPath = []
1105 if searchPaths: 1105 ↛ 1106line 1105 didn't jump to line 1106, because the condition on line 1105 was never true
1106 fullSearchPath.extend(searchPaths)
1108 # Read default paths from enviroment
1109 fullSearchPath.extend(self.defaultSearchPaths())
1111 # There are two places to find defaults for this particular config
1112 # - The "defaultConfigFile" defined in the subclass
1113 # - The class specified in the "cls" element in the config.
1114 # Read cls after merging in case it changes.
1115 if self.defaultConfigFile is not None: 1115 ↛ 1120line 1115 didn't jump to line 1120, because the condition on line 1115 was never false
1116 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile)
1118 # Can have a class specification in the external config (priority)
1119 # or from the defaults.
1120 pytype = None
1121 if "cls" in externalConfig: 1121 ↛ 1122line 1121 didn't jump to line 1122, because the condition on line 1121 was never true
1122 pytype = externalConfig["cls"]
1123 elif "cls" in self: 1123 ↛ 1124line 1123 didn't jump to line 1124, because the condition on line 1123 was never true
1124 pytype = self["cls"]
1126 if pytype is not None: 1126 ↛ 1127line 1126 didn't jump to line 1127, because the condition on line 1126 was never true
1127 try:
1128 cls = doImport(pytype)
1129 except ImportError as e:
1130 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e
1131 defaultsFile = cls.defaultConfigFile
1132 if defaultsFile is not None:
1133 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile)
1135 # Get the container key in case we need it
1136 try:
1137 containerKey = cls.containerKey
1138 except AttributeError:
1139 pass
1141 # Now update this object with the external values so that the external
1142 # values always override the defaults
1143 self.update(externalConfig)
1145 # If this configuration has child configurations of the same
1146 # config class, we need to expand those defaults as well.
1148 if mergeDefaults and containerKey is not None and containerKey in self: 1148 ↛ 1149line 1148 didn't jump to line 1149, because the condition on line 1148 was never true
1149 for idx, subConfig in enumerate(self[containerKey]):
1150 self[containerKey, idx] = type(self)(other=subConfig, validate=validate,
1151 mergeDefaults=mergeDefaults,
1152 searchPaths=searchPaths)
1154 if validate:
1155 self.validate()
1157 @classmethod
1158 def defaultSearchPaths(cls):
1159 """Read environment to determine search paths to use.
1161 Global defaults, at lowest priority, are found in the ``config``
1162 directory of the butler source tree. Additional defaults can be
1163 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS``
1164 which is a PATH-like variable where paths at the front of the list
1165 have priority over those later.
1167 Returns
1168 -------
1169 paths : `list`
1170 Returns a list of paths to search. The returned order is in
1171 priority with the highest priority paths first. The butler config
1172 configuration resources will not be included here but will
1173 always be searched last.
1175 Notes
1176 -----
1177 The environment variable is split on the standard ``:`` path separator.
1178 This currently makes it incompatible with usage of URIs.
1179 """
1180 # We can pick up defaults from multiple search paths
1181 # We fill defaults by using the butler config path and then
1182 # the config path environment variable in reverse order.
1183 defaultsPaths: List[Union[str, ButlerURI]] = []
1185 if CONFIG_PATH in os.environ: 1185 ↛ 1186line 1185 didn't jump to line 1186, because the condition on line 1185 was never true
1186 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep)
1187 defaultsPaths.extend(externalPaths)
1189 # Add the package defaults as a resource
1190 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs",
1191 forceDirectory=True))
1192 return defaultsPaths
1194 def _updateWithConfigsFromPath(self, searchPaths, configFile):
1195 """Search the supplied paths, merging the configuration values.
1197 The values read will override values currently stored in the object.
1198 Every file found in the path will be read, such that the earlier
1199 path entries have higher priority.
1201 Parameters
1202 ----------
1203 searchPaths : `list` of `ButlerURI`, `str`
1204 Paths to search for the supplied configFile. This path
1205 is the priority order, such that files read from the
1206 first path entry will be selected over those read from
1207 a later path. Can contain `str` referring to the local file
1208 system or a URI string.
1209 configFile : `ButlerURI`
1210 File to locate in path. If absolute path it will be read
1211 directly and the search path will not be used. Can be a URI
1212 to an explicit resource (which will ignore the search path)
1213 which is assumed to exist.
1214 """
1215 uri = ButlerURI(configFile)
1216 if uri.isabs() and uri.exists(): 1216 ↛ 1218line 1216 didn't jump to line 1218, because the condition on line 1216 was never true
1217 # Assume this resource exists
1218 self._updateWithOtherConfigFile(configFile)
1219 self.filesRead.append(configFile)
1220 else:
1221 # Reverse order so that high priority entries
1222 # update the object last.
1223 for pathDir in reversed(searchPaths):
1224 if isinstance(pathDir, (str, ButlerURI)): 1224 ↛ 1231line 1224 didn't jump to line 1231, because the condition on line 1224 was never false
1225 pathDir = ButlerURI(pathDir, forceDirectory=True)
1226 file = pathDir.join(configFile)
1227 if file.exists(): 1227 ↛ 1223line 1227 didn't jump to line 1223, because the condition on line 1227 was never false
1228 self.filesRead.append(file)
1229 self._updateWithOtherConfigFile(file)
1230 else:
1231 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}")
1233 def _updateWithOtherConfigFile(self, file):
1234 """Read in some defaults and update.
1236 Update the configuration by reading the supplied file as a config
1237 of this class, and merging such that these values override the
1238 current values. Contents of the external config are not validated.
1240 Parameters
1241 ----------
1242 file : `Config`, `str`, `ButlerURI`, or `dict`
1243 Entity that can be converted to a `ConfigSubset`.
1244 """
1245 # Use this class to read the defaults so that subsetting can happen
1246 # correctly.
1247 externalConfig = type(self)(file, validate=False, mergeDefaults=False)
1248 self.update(externalConfig)
1250 def validate(self):
1251 """Check that mandatory keys are present in this configuration.
1253 Ignored if ``requiredKeys`` is empty.
1254 """
1255 # Validation
1256 missing = [k for k in self.requiredKeys if k not in self._data]
1257 if missing: 1257 ↛ 1258line 1257 didn't jump to line 1258, because the condition on line 1257 was never true
1258 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")