Coverage for python/lsst/daf/butler/core/config.py : 43%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Configuration control."""
26__all__ = ("Config", "ConfigSubset")
28import collections
29import copy
30import json
31import logging
32import pprint
33import os
34import yaml
35import sys
36from pathlib import Path
37from yaml.representer import Representer
38import io
39from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union
41from lsst.utils import doImport
42from ._butlerUri import ButlerURI
44yaml.add_representer(collections.defaultdict, Representer.represent_dict)
47# Config module logger
48log = logging.getLogger(__name__)
50# PATH-like environment variable to use for defaults.
51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH"
53try:
54 yamlLoader = yaml.CSafeLoader
55except AttributeError:
56 # Not all installations have the C library
57 # (but assume for mypy's sake that they're the same)
58 yamlLoader = yaml.SafeLoader # type: ignore
61class Loader(yamlLoader):
62 """YAML Loader that supports file include directives
64 Uses ``!include`` directive in a YAML file to point to another
65 YAML file to be included. The path in the include directive is relative
66 to the file containing that directive.
68 storageClasses: !include storageClasses.yaml
70 Examples
71 --------
72 >>> with open("document.yaml", "r") as f:
73 data = yaml.load(f, Loader=Loader)
75 Notes
76 -----
77 See https://davidchall.github.io/yaml-includes.html
78 """
80 def __init__(self, stream):
81 super().__init__(stream)
82 # if this is a string and not a stream we may well lack a name
83 try:
84 self._root = ButlerURI(stream.name)
85 except AttributeError:
86 # No choice but to assume a local filesystem
87 self._root = ButlerURI("no-file.yaml")
88 Loader.add_constructor("!include", Loader.include)
90 def include(self, node):
91 result: Union[List[Any], Dict[str, Any]]
92 if isinstance(node, yaml.ScalarNode):
93 return self.extractFile(self.construct_scalar(node))
95 elif isinstance(node, yaml.SequenceNode):
96 result = []
97 for filename in self.construct_sequence(node):
98 result.append(self.extractFile(filename))
99 return result
101 elif isinstance(node, yaml.MappingNode):
102 result = {}
103 for k, v in self.construct_mapping(node).items():
104 result[k] = self.extractFile(v)
105 return result
107 else:
108 print("Error:: unrecognised node type in !include statement", file=sys.stderr)
109 raise yaml.constructor.ConstructorError
111 def extractFile(self, filename):
112 # It is possible for the !include to point to an explicit URI
113 # instead of a relative URI, therefore we first see if it is
114 # scheme-less or not. If it has a scheme we use it directly
115 # if it is scheme-less we use it relative to the file root.
116 requesteduri = ButlerURI(filename, forceAbsolute=False)
118 if requesteduri.scheme:
119 fileuri = requesteduri
120 else:
121 fileuri = copy.copy(self._root)
122 fileuri.updateFile(filename)
124 log.debug("Opening YAML file via !include: %s", fileuri)
126 # Read all the data from the resource
127 data = fileuri.read()
129 # Store the bytes into a BytesIO so we can attach a .name
130 stream = io.BytesIO(data)
131 stream.name = fileuri.geturl()
132 return yaml.load(stream, Loader)
135class Config(collections.abc.MutableMapping):
136 r"""Implements a datatype that is used by `Butler` for configuration
137 parameters.
139 It is essentially a `dict` with key/value pairs, including nested dicts
140 (as values). In fact, it can be initialized with a `dict`.
141 This is explained next:
143 Config extends the `dict` api so that hierarchical values may be accessed
144 with delimited notation or as a tuple. If a string is given the delimiter
145 is picked up from the first character in that string. For example,
146 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``,
147 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome.
148 If the first character is alphanumeric, no delimiter will be used.
149 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``.
150 Unicode characters can be used as the delimiter for distinctiveness if
151 required.
153 If a key in the hierarchy starts with a non-alphanumeric character care
154 should be used to ensure that either the tuple interface is used or
155 a distinct delimiter is always given in string form.
157 Finally, the delimiter can be escaped if it is part of a key and also
158 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in
159 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is
160 always better to use a different delimiter in these cases.
162 Note that adding a multi-level key implicitly creates any nesting levels
163 that do not exist, but removing multi-level keys does not automatically
164 remove empty nesting levels. As a result:
166 >>> c = Config()
167 >>> c[".a.b"] = 1
168 >>> del c[".a.b"]
169 >>> c["a"]
170 Config({'a': {}})
172 Storage formats supported:
174 - yaml: read and write is supported.
175 - json: read and write is supported but no ``!include`` directive.
177 Parameters
178 ----------
179 other : `str` or `Config` or `dict` or `ButlerURI` or `pathlib.Path`
180 Other source of configuration, can be:
182 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end
183 with ".yaml".
184 - (`Config`) Copies the other Config's values into this one.
185 - (`dict`) Copies the values from the dict into this Config.
187 If `None` is provided an empty `Config` will be created.
188 """
190 _D: str = "→"
191 """Default internal delimiter to use for components in the hierarchy when
192 constructing keys for external use (see `Config.names()`)."""
194 includeKey: ClassVar[str] = "includeConfigs"
195 """Key used to indicate that another config should be included at this
196 part of the hierarchy."""
198 resourcesPackage: str = "lsst.daf.butler"
199 """Package to search for default configuration data. The resources
200 themselves will be within a ``configs`` resource hierarchy."""
202 def __init__(self, other=None):
203 self._data: Dict[str, Any] = {}
204 self.configFile = None
206 if other is None:
207 return
209 if isinstance(other, Config):
210 self._data = copy.deepcopy(other._data)
211 self.configFile = other.configFile
212 elif isinstance(other, collections.abc.Mapping):
213 self.update(other)
214 elif isinstance(other, (str, ButlerURI, Path)): 214 ↛ 221line 214 didn't jump to line 221, because the condition on line 214 was never false
215 # if other is a string, assume it is a file path/URI
216 self.__initFromUri(other)
217 self._processExplicitIncludes()
218 else:
219 # if the config specified by other could not be recognized raise
220 # a runtime error.
221 raise RuntimeError(f"A Config could not be loaded from other: {other}")
223 def ppprint(self):
224 """helper function for debugging, prints a config out in a readable
225 way in the debugger.
227 use: pdb> print(myConfigObject.ppprint())
229 Returns
230 -------
231 s : `str`
232 A prettyprint formatted string representing the config
233 """
234 return pprint.pformat(self._data, indent=2, width=1)
236 def __repr__(self):
237 return f"{type(self).__name__}({self._data!r})"
239 def __str__(self):
240 return self.ppprint()
242 def __len__(self):
243 return len(self._data)
245 def __iter__(self):
246 return iter(self._data)
248 def copy(self):
249 return type(self)(self)
251 @classmethod
252 def fromString(cls, string: str, format: str = "yaml") -> Config:
253 """Create a new Config instance from a serialized string.
255 Parameters
256 ----------
257 string : `str`
258 String containing content in specified format
259 format : `str`, optional
260 Format of the supplied string. Can be ``json`` or ``yaml``.
262 Returns
263 -------
264 c : `Config`
265 Newly-constructed Config.
266 """
267 if format == "yaml":
268 new_config = cls().__initFromYaml(string)
269 elif format == "json":
270 new_config = cls().__initFromJson(string)
271 else:
272 raise ValueError(f"Unexpected format of string: {format}")
273 new_config._processExplicitIncludes()
274 return new_config
276 @classmethod
277 def fromYaml(cls, string: str) -> Config:
278 """Create a new Config instance from a YAML string.
280 Parameters
281 ----------
282 string : `str`
283 String containing content in YAML format
285 Returns
286 -------
287 c : `Config`
288 Newly-constructed Config.
289 """
290 return cls.fromString(string, format="yaml")
292 def __initFromUri(self, path: Union[str, ButlerURI]) -> None:
293 """Load a file from a path or an URI.
295 Parameters
296 ----------
297 path : `str`
298 Path or a URI to a persisted config file.
299 """
300 uri = ButlerURI(path)
301 ext = uri.getExtension()
302 if ext == ".yaml": 302 ↛ 309line 302 didn't jump to line 309, because the condition on line 302 was never false
303 log.debug("Opening YAML config file: %s", uri.geturl())
304 content = uri.read()
305 # Use a stream so we can name it
306 stream = io.BytesIO(content)
307 stream.name = uri.geturl()
308 self.__initFromYaml(stream)
309 elif ext == ".json":
310 log.debug("Opening JSON config file: %s", uri.geturl())
311 content = uri.read()
312 self.__initFromJson(content)
313 else:
314 # This URI does not have a valid extension. It might be because
315 # we ended up with a directory and not a file. Before we complain
316 # about an extension, do an existence check. No need to do
317 # the (possibly expensive) existence check in the default code
318 # path above because we will find out soon enough that the file
319 # is not there.
320 if not uri.exists():
321 raise FileNotFoundError(f"Config location {uri} does not exist.")
322 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}")
323 self.configFile = uri
325 def __initFromYaml(self, stream):
326 """Loads a YAML config from any readable stream that contains one.
328 Parameters
329 ----------
330 stream: `IO` or `str`
331 Stream to pass to the YAML loader. Accepts anything that
332 `yaml.load` accepts. This can include a string as well as an
333 IO stream.
335 Raises
336 ------
337 yaml.YAMLError
338 If there is an error loading the file.
339 """
340 content = yaml.load(stream, Loader=Loader)
341 if content is None: 341 ↛ 342line 341 didn't jump to line 342, because the condition on line 341 was never true
342 content = {}
343 self._data = content
344 return self
346 def __initFromJson(self, stream):
347 """Loads a JSON config from any readable stream that contains one.
349 Parameters
350 ----------
351 stream: `IO` or `str`
352 Stream to pass to the JSON loader. This can include a string as
353 well as an IO stream.
355 Raises
356 ------
357 TypeError:
358 Raised if there is an error loading the content.
359 """
360 if isinstance(stream, (bytes, str)):
361 content = json.loads(stream)
362 else:
363 content = json.load(stream)
364 if content is None:
365 content = {}
366 self._data = content
367 return self
369 def _processExplicitIncludes(self):
370 """Scan through the configuration searching for the special
371 includeConfigs directive and process the includes."""
373 # Search paths for config files
374 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)]
375 if self.configFile is not None: 375 ↛ 383line 375 didn't jump to line 383, because the condition on line 375 was never false
376 if isinstance(self.configFile, ButlerURI): 376 ↛ 379line 376 didn't jump to line 379, because the condition on line 376 was never false
377 configDir = self.configFile.dirname()
378 else:
379 raise RuntimeError(f"Unexpected type for config file: {self.configFile}")
380 searchPaths.append(configDir)
382 # Ensure we know what delimiter to use
383 names = self.nameTuples()
384 for path in names:
385 if path[-1] == self.includeKey: 385 ↛ 387line 385 didn't jump to line 387, because the condition on line 385 was never true
387 log.debug("Processing file include directive at %s", self._D + self._D.join(path))
388 basePath = path[:-1]
390 # Extract the includes and then delete them from the config
391 includes = self[path]
392 del self[path]
394 # Be consistent and convert to a list
395 if not isinstance(includes, list):
396 includes = [includes]
398 # Read each file assuming it is a reference to a file
399 # The file can be relative to config file or cwd
400 # ConfigSubset search paths are not used
401 subConfigs = []
402 for fileName in includes:
403 # Expand any shell variables -- this could be URI
404 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False)
405 found = None
406 if fileName.isabs():
407 found = fileName
408 else:
409 for dir in searchPaths:
410 if isinstance(dir, ButlerURI):
411 specific = dir.join(fileName.path)
412 # Remote resource check might be expensive
413 if specific.exists():
414 found = specific
415 else:
416 log.warning("Do not understand search path entry '%s' of type %s",
417 dir, type(dir).__name__)
418 if not found:
419 raise RuntimeError(f"Unable to find referenced include file: {fileName}")
421 # Read the referenced Config as a Config
422 subConfigs.append(type(self)(found))
424 # Now we need to merge these sub configs with the current
425 # information that was present in this node in the config
426 # tree with precedence given to the explicit values
427 newConfig = subConfigs.pop(0)
428 for sc in subConfigs:
429 newConfig.update(sc)
431 # Explicit values take precedence
432 if not basePath:
433 # This is an include at the root config
434 newConfig.update(self)
435 # Replace the current config
436 self._data = newConfig._data
437 else:
438 newConfig.update(self[basePath])
439 # And reattach to the base config
440 self[basePath] = newConfig
442 @staticmethod
443 def _splitIntoKeys(key):
444 r"""Split the argument for get/set/in into a hierarchical list.
446 Parameters
447 ----------
448 key : `str` or iterable
449 Argument given to get/set/in. If an iterable is provided it will
450 be converted to a list. If the first character of the string
451 is not an alphanumeric character then it will be used as the
452 delimiter for the purposes of splitting the remainder of the
453 string. If the delimiter is also in one of the keys then it
454 can be escaped using ``\``. There is no default delimiter.
456 Returns
457 -------
458 keys : `list`
459 Hierarchical keys as a `list`.
460 """
461 if isinstance(key, str):
462 if not key[0].isalnum(): 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true
463 d = key[0]
464 key = key[1:]
465 else:
466 return [key, ]
467 escaped = f"\\{d}"
468 temp = None
469 if escaped in key:
470 # Complain at the attempt to escape the escape
471 doubled = fr"\{escaped}"
472 if doubled in key:
473 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})"
474 " is not yet supported.")
475 # Replace with a character that won't be in the string
476 temp = "\r"
477 if temp in key or d == temp:
478 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as"
479 " delimiter if escaping the delimiter")
480 key = key.replace(escaped, temp)
481 hierarchy = key.split(d)
482 if temp:
483 hierarchy = [h.replace(temp, d) for h in hierarchy]
484 return hierarchy
485 elif isinstance(key, collections.abc.Iterable): 485 ↛ 489line 485 didn't jump to line 489, because the condition on line 485 was never false
486 return list(key)
487 else:
488 # Not sure what this is so try it anyway
489 return [key, ]
491 def _getKeyHierarchy(self, name):
492 """Retrieve the key hierarchy for accessing the Config
494 Parameters
495 ----------
496 name : `str` or `tuple`
497 Delimited string or `tuple` of hierarchical keys.
499 Returns
500 -------
501 hierarchy : `list` of `str`
502 Hierarchy to use as a `list`. If the name is available directly
503 as a key in the Config it will be used regardless of the presence
504 of any nominal delimiter.
505 """
506 if name in self._data:
507 keys = [name, ]
508 else:
509 keys = self._splitIntoKeys(name)
510 return keys
512 def _findInHierarchy(self, keys, create=False):
513 """Look for hierarchy of keys in Config
515 Parameters
516 ----------
517 keys : `list` or `tuple`
518 Keys to search in hierarchy.
519 create : `bool`, optional
520 If `True`, if a part of the hierarchy does not exist, insert an
521 empty `dict` into the hierarchy.
523 Returns
524 -------
525 hierarchy : `list`
526 List of the value corresponding to each key in the supplied
527 hierarchy. Only keys that exist in the hierarchy will have
528 a value.
529 complete : `bool`
530 `True` if the full hierarchy exists and the final element
531 in ``hierarchy`` is the value of relevant value.
532 """
533 d = self._data
535 def checkNextItem(k, d, create):
536 """See if k is in d and if it is return the new child"""
537 nextVal = None
538 isThere = False
539 if d is None: 539 ↛ 541line 539 didn't jump to line 541, because the condition on line 539 was never true
540 # We have gone past the end of the hierarchy
541 pass
542 elif isinstance(d, collections.abc.Sequence): 542 ↛ 547line 542 didn't jump to line 547, because the condition on line 542 was never true
543 # Check sequence first because for lists
544 # __contains__ checks whether value is found in list
545 # not whether the index exists in list. When we traverse
546 # the hierarchy we are interested in the index.
547 try:
548 nextVal = d[int(k)]
549 isThere = True
550 except IndexError:
551 pass
552 except ValueError:
553 isThere = k in d
554 elif k in d:
555 nextVal = d[k]
556 isThere = True
557 elif create: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true
558 d[k] = {}
559 nextVal = d[k]
560 isThere = True
561 return nextVal, isThere
563 hierarchy = []
564 complete = True
565 for k in keys:
566 d, isThere = checkNextItem(k, d, create)
567 if isThere:
568 hierarchy.append(d)
569 else:
570 complete = False
571 break
573 return hierarchy, complete
575 def __getitem__(self, name):
576 # Override the split for the simple case where there is an exact
577 # match. This allows `Config.items()` to work via a simple
578 # __iter__ implementation that returns top level keys of
579 # self._data.
580 keys = self._getKeyHierarchy(name)
582 hierarchy, complete = self._findInHierarchy(keys)
583 if not complete: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true
584 raise KeyError(f"{name} not found")
585 data = hierarchy[-1]
587 if isinstance(data, collections.abc.Mapping):
588 data = Config(data)
589 # Ensure that child configs inherit the parent internal delimiter
590 if self._D != Config._D: 590 ↛ 591line 590 didn't jump to line 591, because the condition on line 590 was never true
591 data._D = self._D
592 return data
594 def __setitem__(self, name, value):
595 keys = self._getKeyHierarchy(name)
596 last = keys.pop()
597 if isinstance(value, Config):
598 value = copy.deepcopy(value._data)
600 hierarchy, complete = self._findInHierarchy(keys, create=True)
601 if hierarchy:
602 data = hierarchy[-1]
603 else:
604 data = self._data
606 try:
607 data[last] = value
608 except TypeError:
609 data[int(last)] = value
611 def __contains__(self, key):
612 keys = self._getKeyHierarchy(key)
613 hierarchy, complete = self._findInHierarchy(keys)
614 return complete
616 def __delitem__(self, key):
617 keys = self._getKeyHierarchy(key)
618 last = keys.pop()
619 hierarchy, complete = self._findInHierarchy(keys)
620 if complete: 620 ↛ 627line 620 didn't jump to line 627, because the condition on line 620 was never false
621 if hierarchy: 621 ↛ 622line 621 didn't jump to line 622, because the condition on line 621 was never true
622 data = hierarchy[-1]
623 else:
624 data = self._data
625 del data[last]
626 else:
627 raise KeyError(f"{key} not found in Config")
629 def update(self, other):
630 """Like dict.update, but will add or modify keys in nested dicts,
631 instead of overwriting the nested dict entirely.
633 For example, for the given code:
634 foo = {"a": {"b": 1}}
635 foo.update({"a": {"c": 2}})
637 Parameters
638 ----------
639 other : `dict` or `Config`
640 Source of configuration:
642 - If foo is a dict, then after the update foo == {"a": {"c": 2}}
643 - But if foo is a Config, then after the update
644 foo == {"a": {"b": 1, "c": 2}}
645 """
646 def doUpdate(d, u):
647 if not isinstance(u, collections.abc.Mapping) or \ 647 ↛ 649line 647 didn't jump to line 649, because the condition on line 647 was never true
648 not isinstance(d, collections.abc.MutableMapping):
649 raise RuntimeError("Only call update with Mapping, not {}".format(type(d)))
650 for k, v in u.items():
651 if isinstance(v, collections.abc.Mapping):
652 d[k] = doUpdate(d.get(k, {}), v)
653 else:
654 d[k] = v
655 return d
656 doUpdate(self._data, other)
658 def merge(self, other):
659 """Like Config.update, but will add keys & values from other that
660 DO NOT EXIST in self.
662 Keys and values that already exist in self will NOT be overwritten.
664 Parameters
665 ----------
666 other : `dict` or `Config`
667 Source of configuration:
668 """
669 otherCopy = copy.deepcopy(other)
670 otherCopy.update(self)
671 self._data = otherCopy._data
673 def nameTuples(self, topLevelOnly=False):
674 """Get tuples representing the name hierarchies of all keys.
676 The tuples returned from this method are guaranteed to be usable
677 to access items in the configuration object.
679 Parameters
680 ----------
681 topLevelOnly : `bool`, optional
682 If False, the default, a full hierarchy of names is returned.
683 If True, only the top level are returned.
685 Returns
686 -------
687 names : `list` of `tuple` of `str`
688 List of all names present in the `Config` where each element
689 in the list is a `tuple` of strings representing the hierarchy.
690 """
691 if topLevelOnly: 691 ↛ 692line 691 didn't jump to line 692, because the condition on line 691 was never true
692 return list((k,) for k in self)
694 def getKeysAsTuples(d, keys, base):
695 if isinstance(d, collections.abc.Sequence):
696 theseKeys = range(len(d))
697 else:
698 theseKeys = d.keys()
699 for key in theseKeys:
700 val = d[key]
701 levelKey = base + (key,) if base is not None else (key,)
702 keys.append(levelKey)
703 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \
704 and not isinstance(val, str):
705 getKeysAsTuples(val, keys, levelKey)
706 keys: List[Tuple[str, ...]] = []
707 getKeysAsTuples(self._data, keys, None)
708 return keys
710 def names(self, topLevelOnly=False, delimiter=None):
711 """Get a delimited name of all the keys in the hierarchy.
713 The values returned from this method are guaranteed to be usable
714 to access items in the configuration object.
716 Parameters
717 ----------
718 topLevelOnly : `bool`, optional
719 If False, the default, a full hierarchy of names is returned.
720 If True, only the top level are returned.
721 delimiter : `str`, optional
722 Delimiter to use when forming the keys. If the delimiter is
723 present in any of the keys, it will be escaped in the returned
724 names. If `None` given a delimiter will be automatically provided.
725 The delimiter can not be alphanumeric.
727 Returns
728 -------
729 names : `list` of `str`
730 List of all names present in the `Config`.
732 Notes
733 -----
734 This is different than the built-in method `dict.keys`, which will
735 return only the first level keys.
737 Raises
738 ------
739 ValueError:
740 The supplied delimiter is alphanumeric.
741 """
742 if topLevelOnly:
743 return list(self.keys())
745 # Get all the tuples of hierarchical keys
746 nameTuples = self.nameTuples()
748 if delimiter is not None and delimiter.isalnum():
749 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.")
751 if delimiter is None:
752 # Start with something, and ensure it does not need to be
753 # escaped (it is much easier to understand if not escaped)
754 delimiter = self._D
756 # Form big string for easy check of delimiter clash
757 combined = "".join("".join(str(s) for s in k) for k in nameTuples)
759 # Try a delimiter and keep trying until we get something that
760 # works.
761 ntries = 0
762 while delimiter in combined:
763 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.")
764 ntries += 1
766 if ntries > 100:
767 raise ValueError(f"Unable to determine a delimiter for Config {self}")
769 # try another one
770 while True:
771 delimiter = chr(ord(delimiter)+1)
772 if not delimiter.isalnum():
773 break
775 log.debug(f"Using delimiter {delimiter!r}")
777 # Form the keys, escaping the delimiter if necessary
778 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k)
779 for k in nameTuples]
780 return strings
782 def asArray(self, name):
783 """Get a value as an array.
785 May contain one or more elements.
787 Parameters
788 ----------
789 name : `str`
790 Key to use to retrieve value.
792 Returns
793 -------
794 array : `collections.abc.Sequence`
795 The value corresponding to name, but guaranteed to be returned
796 as a list with at least one element. If the value is a
797 `~collections.abc.Sequence` (and not a `str`) the value itself
798 will be returned, else the value will be the first element.
799 """
800 val = self.get(name)
801 if isinstance(val, str):
802 val = [val]
803 elif not isinstance(val, collections.abc.Sequence):
804 val = [val]
805 return val
807 def __eq__(self, other):
808 if isinstance(other, Config):
809 other = other._data
810 return self._data == other
812 def __ne__(self, other):
813 if isinstance(other, Config):
814 other = other._data
815 return self._data != other
817 #######
818 # i/o #
820 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]:
821 """Writes the config to an output stream.
823 Parameters
824 ----------
825 output : `IO`, optional
826 The stream to use for output. If `None` the serialized content
827 will be returned.
828 format : `str`, optional
829 The format to use for the output. Can be "yaml" or "json".
831 Returns
832 -------
833 serialized : `str` or `None`
834 If a stream was given the stream will be used and the return
835 value will be `None`. If the stream was `None` the
836 serialization will be returned as a string.
837 """
838 if format == "yaml":
839 return yaml.safe_dump(self._data, output, default_flow_style=False)
840 elif format == "json":
841 if output is not None:
842 json.dump(self._data, output, ensure_ascii=False)
843 return None
844 else:
845 return json.dumps(self._data, ensure_ascii=False)
846 raise ValueError(f"Unsupported format for Config serialization: {format}")
848 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
849 defaultFileName: str = "butler.yaml",
850 overwrite: bool = True) -> None:
851 """Writes the config to location pointed to by given URI.
853 Currently supports 's3' and 'file' URI schemes.
855 Parameters
856 ----------
857 uri: `str` or `ButlerURI`
858 URI of location where the Config will be written.
859 updateFile : bool, optional
860 If True and uri does not end on a filename with extension, will
861 append `defaultFileName` to the target uri. True by default.
862 defaultFileName : bool, optional
863 The file name that will be appended to target uri if updateFile is
864 True and uri does not end on a file with an extension.
865 overwrite : bool, optional
866 If True the configuration will be written even if it already
867 exists at that location.
868 """
869 # Make local copy of URI or create new one
870 uri = ButlerURI(uri)
872 if updateFile and not uri.getExtension():
873 uri.updateFile(defaultFileName)
875 # Try to work out the format from the extension
876 ext = uri.getExtension()
877 format = ext[1:].lower()
879 output = self.dump(format=format)
880 assert output is not None, "Config.dump guarantees not-None return when output arg is None"
881 uri.write(output.encode(), overwrite=overwrite)
882 self.configFile = uri
884 @staticmethod
885 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True):
886 """Generic helper function for updating specific config parameters.
888 Allows for named parameters to be set to new values in bulk, and
889 for other values to be set by copying from a reference config.
891 Assumes that the supplied config is compatible with ``configType``
892 and will attach the updated values to the supplied config by
893 looking for the related component key. It is assumed that
894 ``config`` and ``full`` are from the same part of the
895 configuration hierarchy.
897 Parameters
898 ----------
899 configType : `ConfigSubset`
900 Config type to use to extract relevant items from ``config``.
901 config : `Config`
902 A `Config` to update. Only the subset understood by
903 the supplied `ConfigSubset` will be modified. Default values
904 will not be inserted and the content will not be validated
905 since mandatory keys are allowed to be missing until
906 populated later by merging.
907 full : `Config`
908 A complete config with all defaults expanded that can be
909 converted to a ``configType``. Read-only and will not be
910 modified by this method. Values are read from here if
911 ``toCopy`` is defined.
913 Repository-specific options that should not be obtained
914 from defaults when Butler instances are constructed
915 should be copied from ``full`` to ``config``.
916 toUpdate : `dict`, optional
917 A `dict` defining the keys to update and the new value to use.
918 The keys and values can be any supported by `Config`
919 assignment.
920 toCopy : `tuple`, optional
921 `tuple` of keys whose values should be copied from ``full``
922 into ``config``.
923 overwrite : `bool`, optional
924 If `False`, do not modify a value in ``config`` if the key
925 already exists. Default is always to overwrite.
927 Raises
928 ------
929 ValueError
930 Neither ``toUpdate`` not ``toCopy`` were defined.
931 """
932 if toUpdate is None and toCopy is None:
933 raise ValueError("One of toUpdate or toCopy parameters must be set.")
935 # If this is a parent configuration then we need to ensure that
936 # the supplied config has the relevant component key in it.
937 # If this is a parent configuration we add in the stub entry
938 # so that the ConfigSubset constructor will do the right thing.
939 # We check full for this since that is guaranteed to be complete.
940 if configType.component in full and configType.component not in config:
941 config[configType.component] = {}
943 # Extract the part of the config we wish to update
944 localConfig = configType(config, mergeDefaults=False, validate=False)
946 if toUpdate:
947 for key, value in toUpdate.items():
948 if key in localConfig and not overwrite:
949 log.debug("Not overriding key '%s' with value '%s' in config %s",
950 key, value, localConfig.__class__.__name__)
951 else:
952 localConfig[key] = value
954 if toCopy:
955 localFullConfig = configType(full, mergeDefaults=False)
956 for key in toCopy:
957 if key in localConfig and not overwrite:
958 log.debug("Not overriding key '%s' from defaults in config %s",
959 key, localConfig.__class__.__name__)
960 else:
961 localConfig[key] = localFullConfig[key]
963 # Reattach to parent if this is a child config
964 if configType.component in config:
965 config[configType.component] = localConfig
966 else:
967 config.update(localConfig)
969 def toDict(self):
970 """Convert a `Config` to a standalone hierarchical `dict`.
972 Returns
973 -------
974 d : `dict`
975 The standalone hierarchical `dict` with any `Config` classes
976 in the hierarchy converted to `dict`.
978 Notes
979 -----
980 This can be useful when passing a Config to some code that
981 expects native Python types.
982 """
983 output = copy.deepcopy(self._data)
984 for k, v in output.items():
985 if isinstance(v, Config):
986 v = v.toDict()
987 output[k] = v
988 return output
991class ConfigSubset(Config):
992 """Config representing a subset of a more general configuration.
994 Subclasses define their own component and when given a configuration
995 that includes that component, the resulting configuration only includes
996 the subset. For example, your config might contain ``dimensions`` if it's
997 part of a global config and that subset will be stored. If ``dimensions``
998 can not be found it is assumed that the entire contents of the
999 configuration should be used.
1001 Default values are read from the environment or supplied search paths
1002 using the default configuration file name specified in the subclass.
1003 This allows a configuration class to be instantiated without any
1004 additional arguments.
1006 Additional validation can be specified to check for keys that are mandatory
1007 in the configuration.
1009 Parameters
1010 ----------
1011 other : `Config` or `str` or `dict`
1012 Argument specifying the configuration information as understood
1013 by `Config`
1014 validate : `bool`, optional
1015 If `True` required keys will be checked to ensure configuration
1016 consistency.
1017 mergeDefaults : `bool`, optional
1018 If `True` defaults will be read and the supplied config will
1019 be combined with the defaults, with the supplied valiues taking
1020 precedence.
1021 searchPaths : `list` or `tuple`, optional
1022 Explicit additional paths to search for defaults. They should
1023 be supplied in priority order. These paths have higher priority
1024 than those read from the environment in
1025 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
1026 the local file system or URIs, `ButlerURI`.
1027 """
1029 component: ClassVar[Optional[str]] = None
1030 """Component to use from supplied config. Can be None. If specified the
1031 key is not required. Can be a full dot-separated path to a component.
1032 """
1034 requiredKeys: ClassVar[Sequence[str]] = ()
1035 """Keys that are required to be specified in the configuration.
1036 """
1038 defaultConfigFile: ClassVar[Optional[str]] = None
1039 """Name of the file containing defaults for this config class.
1040 """
1042 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None):
1044 # Create a blank object to receive the defaults
1045 # Once we have the defaults we then update with the external values
1046 super().__init__()
1048 # Create a standard Config rather than subset
1049 externalConfig = Config(other)
1051 # Select the part we need from it
1052 # To simplify the use of !include we also check for the existence of
1053 # component.component (since the included files can themselves
1054 # include the component name)
1055 if self.component is not None: 1055 ↛ 1064line 1055 didn't jump to line 1064, because the condition on line 1055 was never false
1056 doubled = (self.component, self.component)
1057 # Must check for double depth first
1058 if doubled in externalConfig: 1058 ↛ 1059line 1058 didn't jump to line 1059, because the condition on line 1058 was never true
1059 externalConfig = externalConfig[doubled]
1060 elif self.component in externalConfig:
1061 externalConfig._data = externalConfig._data[self.component]
1063 # Default files read to create this configuration
1064 self.filesRead = []
1066 # Assume we are not looking up child configurations
1067 containerKey = None
1069 # Sometimes we do not want to merge with defaults.
1070 if mergeDefaults:
1072 # Supplied search paths have highest priority
1073 fullSearchPath = []
1074 if searchPaths: 1074 ↛ 1075line 1074 didn't jump to line 1075, because the condition on line 1074 was never true
1075 fullSearchPath.extend(searchPaths)
1077 # Read default paths from enviroment
1078 fullSearchPath.extend(self.defaultSearchPaths())
1080 # There are two places to find defaults for this particular config
1081 # - The "defaultConfigFile" defined in the subclass
1082 # - The class specified in the "cls" element in the config.
1083 # Read cls after merging in case it changes.
1084 if self.defaultConfigFile is not None: 1084 ↛ 1089line 1084 didn't jump to line 1089, because the condition on line 1084 was never false
1085 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile)
1087 # Can have a class specification in the external config (priority)
1088 # or from the defaults.
1089 pytype = None
1090 if "cls" in externalConfig: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the condition on line 1090 was never true
1091 pytype = externalConfig["cls"]
1092 elif "cls" in self: 1092 ↛ 1093line 1092 didn't jump to line 1093, because the condition on line 1092 was never true
1093 pytype = self["cls"]
1095 if pytype is not None: 1095 ↛ 1096line 1095 didn't jump to line 1096, because the condition on line 1095 was never true
1096 try:
1097 cls = doImport(pytype)
1098 except ImportError as e:
1099 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e
1100 defaultsFile = cls.defaultConfigFile
1101 if defaultsFile is not None:
1102 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile)
1104 # Get the container key in case we need it
1105 try:
1106 containerKey = cls.containerKey
1107 except AttributeError:
1108 pass
1110 # Now update this object with the external values so that the external
1111 # values always override the defaults
1112 self.update(externalConfig)
1114 # If this configuration has child configurations of the same
1115 # config class, we need to expand those defaults as well.
1117 if mergeDefaults and containerKey is not None and containerKey in self: 1117 ↛ 1118line 1117 didn't jump to line 1118, because the condition on line 1117 was never true
1118 for idx, subConfig in enumerate(self[containerKey]):
1119 self[containerKey, idx] = type(self)(other=subConfig, validate=validate,
1120 mergeDefaults=mergeDefaults,
1121 searchPaths=searchPaths)
1123 if validate:
1124 self.validate()
1126 @classmethod
1127 def defaultSearchPaths(cls):
1128 """Read the environment to determine search paths to use for global
1129 defaults.
1131 Global defaults, at lowest priority, are found in the ``config``
1132 directory of the butler source tree. Additional defaults can be
1133 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS``
1134 which is a PATH-like variable where paths at the front of the list
1135 have priority over those later.
1137 Returns
1138 -------
1139 paths : `list`
1140 Returns a list of paths to search. The returned order is in
1141 priority with the highest priority paths first. The butler config
1142 configuration resources will not be included here but will
1143 always be searched last.
1145 Notes
1146 -----
1147 The environment variable is split on the standard ``:`` path separator.
1148 This currently makes it incompatible with usage of URIs.
1149 """
1150 # We can pick up defaults from multiple search paths
1151 # We fill defaults by using the butler config path and then
1152 # the config path environment variable in reverse order.
1153 defaultsPaths: List[Union[str, ButlerURI]] = []
1155 if CONFIG_PATH in os.environ: 1155 ↛ 1156line 1155 didn't jump to line 1156, because the condition on line 1155 was never true
1156 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep)
1157 defaultsPaths.extend(externalPaths)
1159 # Add the package defaults as a resource
1160 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs",
1161 forceDirectory=True))
1162 return defaultsPaths
1164 def _updateWithConfigsFromPath(self, searchPaths, configFile):
1165 """Search the supplied paths, merging the configuration values
1167 The values read will override values currently stored in the object.
1168 Every file found in the path will be read, such that the earlier
1169 path entries have higher priority.
1171 Parameters
1172 ----------
1173 searchPaths : `list` of `ButlerURI`, `str`
1174 Paths to search for the supplied configFile. This path
1175 is the priority order, such that files read from the
1176 first path entry will be selected over those read from
1177 a later path. Can contain `str` referring to the local file
1178 system or a URI string.
1179 configFile : `ButlerURI`
1180 File to locate in path. If absolute path it will be read
1181 directly and the search path will not be used. Can be a URI
1182 to an explicit resource (which will ignore the search path)
1183 which is assumed to exist.
1184 """
1185 uri = ButlerURI(configFile)
1186 if uri.isabs() and uri.exists(): 1186 ↛ 1188line 1186 didn't jump to line 1188, because the condition on line 1186 was never true
1187 # Assume this resource exists
1188 self._updateWithOtherConfigFile(configFile)
1189 self.filesRead.append(configFile)
1190 else:
1191 # Reverse order so that high priority entries
1192 # update the object last.
1193 for pathDir in reversed(searchPaths):
1194 if isinstance(pathDir, (str, ButlerURI)): 1194 ↛ 1201line 1194 didn't jump to line 1201, because the condition on line 1194 was never false
1195 pathDir = ButlerURI(pathDir, forceDirectory=True)
1196 file = pathDir.join(configFile)
1197 if file.exists(): 1197 ↛ 1193line 1197 didn't jump to line 1193, because the condition on line 1197 was never false
1198 self.filesRead.append(file)
1199 self._updateWithOtherConfigFile(file)
1200 else:
1201 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}")
1203 def _updateWithOtherConfigFile(self, file):
1204 """Read in some defaults and update.
1206 Update the configuration by reading the supplied file as a config
1207 of this class, and merging such that these values override the
1208 current values. Contents of the external config are not validated.
1210 Parameters
1211 ----------
1212 file : `Config`, `str`, `ButlerURI`, or `dict`
1213 Entity that can be converted to a `ConfigSubset`.
1214 """
1215 # Use this class to read the defaults so that subsetting can happen
1216 # correctly.
1217 externalConfig = type(self)(file, validate=False, mergeDefaults=False)
1218 self.update(externalConfig)
1220 def validate(self):
1221 """Check that mandatory keys are present in this configuration.
1223 Ignored if ``requiredKeys`` is empty."""
1224 # Validation
1225 missing = [k for k in self.requiredKeys if k not in self._data]
1226 if missing: 1226 ↛ 1227line 1226 didn't jump to line 1227, because the condition on line 1226 was never true
1227 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")