Coverage for python/lsst/daf/butler/core/config.py : 43%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""Configuration control."""
26__all__ = ("Config", "ConfigSubset")
28import collections
29import copy
30import json
31import logging
32import pprint
33import os
34import yaml
35import sys
36from yaml.representer import Representer
37import io
38from typing import Sequence, Optional, ClassVar, IO, Union
40from lsst.utils import doImport
41from ._butlerUri import ButlerURI
43yaml.add_representer(collections.defaultdict, Representer.represent_dict)
46# Config module logger
47log = logging.getLogger(__name__)
49# PATH-like environment variable to use for defaults.
50CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH"
52try:
53 yamlLoader = yaml.CSafeLoader
54except AttributeError:
55 # Not all installations have the C library
56 yamlLoader = yaml.SafeLoader
59class Loader(yamlLoader):
60 """YAML Loader that supports file include directives
62 Uses ``!include`` directive in a YAML file to point to another
63 YAML file to be included. The path in the include directive is relative
64 to the file containing that directive.
66 storageClasses: !include storageClasses.yaml
68 Examples
69 --------
70 >>> with open("document.yaml", "r") as f:
71 data = yaml.load(f, Loader=Loader)
73 Notes
74 -----
75 See https://davidchall.github.io/yaml-includes.html
76 """
78 def __init__(self, stream):
79 super().__init__(stream)
80 # if this is a string and not a stream we may well lack a name
81 try:
82 self._root = ButlerURI(stream.name)
83 except AttributeError:
84 # No choice but to assume a local filesystem
85 self._root = ButlerURI("no-file.yaml")
86 Loader.add_constructor("!include", Loader.include)
88 def include(self, node):
89 if isinstance(node, yaml.ScalarNode):
90 return self.extractFile(self.construct_scalar(node))
92 elif isinstance(node, yaml.SequenceNode):
93 result = []
94 for filename in self.construct_sequence(node):
95 result.append(self.extractFile(filename))
96 return result
98 elif isinstance(node, yaml.MappingNode):
99 result = {}
100 for k, v in self.construct_mapping(node).items():
101 result[k] = self.extractFile(v)
102 return result
104 else:
105 print("Error:: unrecognised node type in !include statement", file=sys.stderr)
106 raise yaml.constructor.ConstructorError
108 def extractFile(self, filename):
109 # It is possible for the !include to point to an explicit URI
110 # instead of a relative URI, therefore we first see if it is
111 # scheme-less or not. If it has a scheme we use it directly
112 # if it is scheme-less we use it relative to the file root.
113 requesteduri = ButlerURI(filename, forceAbsolute=False)
115 if requesteduri.scheme:
116 fileuri = requesteduri
117 else:
118 fileuri = copy.copy(self._root)
119 fileuri.updateFile(filename)
121 log.debug("Opening YAML file via !include: %s", fileuri)
123 # Read all the data from the resource
124 data = fileuri.read()
126 # Store the bytes into a BytesIO so we can attach a .name
127 stream = io.BytesIO(data)
128 stream.name = fileuri.geturl()
129 return yaml.load(stream, Loader)
132class Config(collections.abc.MutableMapping):
133 r"""Implements a datatype that is used by `Butler` for configuration
134 parameters.
136 It is essentially a `dict` with key/value pairs, including nested dicts
137 (as values). In fact, it can be initialized with a `dict`.
138 This is explained next:
140 Config extends the `dict` api so that hierarchical values may be accessed
141 with delimited notation or as a tuple. If a string is given the delimiter
142 is picked up from the first character in that string. For example,
143 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``,
144 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome.
145 If the first character is alphanumeric, no delimiter will be used.
146 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``.
147 Unicode characters can be used as the delimiter for distinctiveness if
148 required.
150 If a key in the hierarchy starts with a non-alphanumeric character care
151 should be used to ensure that either the tuple interface is used or
152 a distinct delimiter is always given in string form.
154 Finally, the delimiter can be escaped if it is part of a key and also
155 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in
156 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is
157 always better to use a different delimiter in these cases.
159 Note that adding a multi-level key implicitly creates any nesting levels
160 that do not exist, but removing multi-level keys does not automatically
161 remove empty nesting levels. As a result:
163 >>> c = Config()
164 >>> c[".a.b"] = 1
165 >>> del c[".a.b"]
166 >>> c["a"]
167 Config({'a': {}})
169 Storage formats supported:
171 - yaml: read and write is supported.
172 - json: read and write is supported but no ``!include`` directive.
174 Parameters
175 ----------
176 other : `str` or `Config` or `dict` or `ButlerURI`
177 Other source of configuration, can be:
179 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end
180 with ".yaml".
181 - (`Config`) Copies the other Config's values into this one.
182 - (`dict`) Copies the values from the dict into this Config.
184 If `None` is provided an empty `Config` will be created.
185 """
187 _D: ClassVar[str] = "→"
188 """Default internal delimiter to use for components in the hierarchy when
189 constructing keys for external use (see `Config.names()`)."""
191 includeKey: ClassVar[str] = "includeConfigs"
192 """Key used to indicate that another config should be included at this
193 part of the hierarchy."""
195 resourcesPackage: str = "lsst.daf.butler"
196 """Package to search for default configuration data. The resources
197 themselves will be within a ``configs`` resource hierarchy."""
199 def __init__(self, other=None):
200 self._data = {}
201 self.configFile = None
203 if other is None:
204 return
206 if isinstance(other, Config):
207 self._data = copy.deepcopy(other._data)
208 self.configFile = other.configFile
209 elif isinstance(other, collections.abc.Mapping):
210 self.update(other)
211 elif isinstance(other, (str, ButlerURI)): 211 ↛ 218line 211 didn't jump to line 218, because the condition on line 211 was never false
212 # if other is a string, assume it is a file path/URI
213 self.__initFromUri(other)
214 self._processExplicitIncludes()
215 else:
216 # if the config specified by other could not be recognized raise
217 # a runtime error.
218 raise RuntimeError(f"A Config could not be loaded from other: {other}")
220 def ppprint(self):
221 """helper function for debugging, prints a config out in a readable
222 way in the debugger.
224 use: pdb> print(myConfigObject.ppprint())
226 Returns
227 -------
228 s : `str`
229 A prettyprint formatted string representing the config
230 """
231 return pprint.pformat(self._data, indent=2, width=1)
233 def __repr__(self):
234 return f"{type(self).__name__}({self._data!r})"
236 def __str__(self):
237 return self.ppprint()
239 def __len__(self):
240 return len(self._data)
242 def __iter__(self):
243 return iter(self._data)
245 def copy(self):
246 return type(self)(self)
248 @classmethod
249 def fromString(cls, string: str, format: str = "yaml") -> Config:
250 """Create a new Config instance from a serialized string.
252 Parameters
253 ----------
254 string : `str`
255 String containing content in specified format
256 format : `str`, optional
257 Format of the supplied string. Can be ``json`` or ``yaml``.
259 Returns
260 -------
261 c : `Config`
262 Newly-constructed Config.
263 """
264 if format == "yaml":
265 new_config = cls().__initFromYaml(string)
266 elif format == "json":
267 new_config = cls().__initFromJson(string)
268 else:
269 raise ValueError(f"Unexpected format of string: {format}")
270 new_config._processExplicitIncludes()
271 return new_config
273 @classmethod
274 def fromYaml(cls, string: str) -> Config:
275 """Create a new Config instance from a YAML string.
277 Parameters
278 ----------
279 string : `str`
280 String containing content in YAML format
282 Returns
283 -------
284 c : `Config`
285 Newly-constructed Config.
286 """
287 return cls.fromString(string, format="yaml")
289 def __initFromUri(self, path: str) -> None:
290 """Load a file from a path or an URI.
292 Parameters
293 ----------
294 path : `str`
295 Path or a URI to a persisted config file.
296 """
297 uri = ButlerURI(path)
298 ext = uri.getExtension()
299 if ext == ".yaml": 299 ↛ 306line 299 didn't jump to line 306, because the condition on line 299 was never false
300 log.debug("Opening YAML config file: %s", uri.geturl())
301 content = uri.read()
302 # Use a stream so we can name it
303 stream = io.BytesIO(content)
304 stream.name = uri.geturl()
305 self.__initFromYaml(stream)
306 elif ext == ".json":
307 log.debug("Opening JSON config file: %s", uri.geturl())
308 content = uri.read()
309 self.__initFromJson(content)
310 else:
311 # This URI does not have a valid extension. It might be because
312 # we ended up with a directory and not a file. Before we complain
313 # about an extension, do an existence check. No need to do
314 # the (possibly expensive) existence check in the default code
315 # path above because we will find out soon enough that the file
316 # is not there.
317 if not uri.exists():
318 raise FileNotFoundError(f"Config location {uri} does not exist.")
319 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}")
320 self.configFile = uri
322 def __initFromYaml(self, stream):
323 """Loads a YAML config from any readable stream that contains one.
325 Parameters
326 ----------
327 stream: `IO` or `str`
328 Stream to pass to the YAML loader. Accepts anything that
329 `yaml.load` accepts. This can include a string as well as an
330 IO stream.
332 Raises
333 ------
334 yaml.YAMLError
335 If there is an error loading the file.
336 """
337 content = yaml.load(stream, Loader=Loader)
338 if content is None: 338 ↛ 339line 338 didn't jump to line 339, because the condition on line 338 was never true
339 content = {}
340 self._data = content
341 return self
343 def __initFromJson(self, stream):
344 """Loads a JSON config from any readable stream that contains one.
346 Parameters
347 ----------
348 stream: `IO` or `str`
349 Stream to pass to the JSON loader. This can include a string as
350 well as an IO stream.
352 Raises
353 ------
354 TypeError:
355 Raised if there is an error loading the content.
356 """
357 if isinstance(stream, (bytes, str)):
358 content = json.loads(stream)
359 else:
360 content = json.load(stream)
361 if content is None:
362 content = {}
363 self._data = content
364 return self
366 def _processExplicitIncludes(self):
367 """Scan through the configuration searching for the special
368 includeConfigs directive and process the includes."""
370 # Search paths for config files
371 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)]
372 if self.configFile is not None: 372 ↛ 380line 372 didn't jump to line 380, because the condition on line 372 was never false
373 if isinstance(self.configFile, ButlerURI): 373 ↛ 376line 373 didn't jump to line 376, because the condition on line 373 was never false
374 configDir = self.configFile.dirname()
375 else:
376 raise RuntimeError(f"Unexpected type for config file: {self.configFile}")
377 searchPaths.append(configDir)
379 # Ensure we know what delimiter to use
380 names = self.nameTuples()
381 for path in names:
382 if path[-1] == self.includeKey: 382 ↛ 384line 382 didn't jump to line 384, because the condition on line 382 was never true
384 log.debug("Processing file include directive at %s", self._D + self._D.join(path))
385 basePath = path[:-1]
387 # Extract the includes and then delete them from the config
388 includes = self[path]
389 del self[path]
391 # Be consistent and convert to a list
392 if not isinstance(includes, list):
393 includes = [includes]
395 # Read each file assuming it is a reference to a file
396 # The file can be relative to config file or cwd
397 # ConfigSubset search paths are not used
398 subConfigs = []
399 for fileName in includes:
400 # Expand any shell variables -- this could be URI
401 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False)
402 found = None
403 if fileName.isabs():
404 found = fileName
405 else:
406 for dir in searchPaths:
407 if isinstance(dir, ButlerURI):
408 specific = dir.join(fileName.path)
409 # Remote resource check might be expensive
410 if specific.exists():
411 found = specific
412 else:
413 log.warning("Do not understand search path entry '%s' of type %s",
414 dir, type(dir).__name__)
415 if not found:
416 raise RuntimeError(f"Unable to find referenced include file: {fileName}")
418 # Read the referenced Config as a Config
419 subConfigs.append(type(self)(found))
421 # Now we need to merge these sub configs with the current
422 # information that was present in this node in the config
423 # tree with precedence given to the explicit values
424 newConfig = subConfigs.pop(0)
425 for sc in subConfigs:
426 newConfig.update(sc)
428 # Explicit values take precedence
429 if not basePath:
430 # This is an include at the root config
431 newConfig.update(self)
432 # Replace the current config
433 self._data = newConfig._data
434 else:
435 newConfig.update(self[basePath])
436 # And reattach to the base config
437 self[basePath] = newConfig
439 @staticmethod
440 def _splitIntoKeys(key):
441 r"""Split the argument for get/set/in into a hierarchical list.
443 Parameters
444 ----------
445 key : `str` or iterable
446 Argument given to get/set/in. If an iterable is provided it will
447 be converted to a list. If the first character of the string
448 is not an alphanumeric character then it will be used as the
449 delimiter for the purposes of splitting the remainder of the
450 string. If the delimiter is also in one of the keys then it
451 can be escaped using ``\``. There is no default delimiter.
453 Returns
454 -------
455 keys : `list`
456 Hierarchical keys as a `list`.
457 """
458 if isinstance(key, str):
459 if not key[0].isalnum(): 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true
460 d = key[0]
461 key = key[1:]
462 else:
463 return [key, ]
464 escaped = f"\\{d}"
465 temp = None
466 if escaped in key:
467 # Complain at the attempt to escape the escape
468 doubled = fr"\{escaped}"
469 if doubled in key:
470 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})"
471 " is not yet supported.")
472 # Replace with a character that won't be in the string
473 temp = "\r"
474 if temp in key or d == temp:
475 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as"
476 " delimiter if escaping the delimiter")
477 key = key.replace(escaped, temp)
478 hierarchy = key.split(d)
479 if temp:
480 hierarchy = [h.replace(temp, d) for h in hierarchy]
481 return hierarchy
482 elif isinstance(key, collections.abc.Iterable): 482 ↛ 486line 482 didn't jump to line 486, because the condition on line 482 was never false
483 return list(key)
484 else:
485 # Not sure what this is so try it anyway
486 return [key, ]
488 def _getKeyHierarchy(self, name):
489 """Retrieve the key hierarchy for accessing the Config
491 Parameters
492 ----------
493 name : `str` or `tuple`
494 Delimited string or `tuple` of hierarchical keys.
496 Returns
497 -------
498 hierarchy : `list` of `str`
499 Hierarchy to use as a `list`. If the name is available directly
500 as a key in the Config it will be used regardless of the presence
501 of any nominal delimiter.
502 """
503 if name in self._data:
504 keys = [name, ]
505 else:
506 keys = self._splitIntoKeys(name)
507 return keys
509 def _findInHierarchy(self, keys, create=False):
510 """Look for hierarchy of keys in Config
512 Parameters
513 ----------
514 keys : `list` or `tuple`
515 Keys to search in hierarchy.
516 create : `bool`, optional
517 If `True`, if a part of the hierarchy does not exist, insert an
518 empty `dict` into the hierarchy.
520 Returns
521 -------
522 hierarchy : `list`
523 List of the value corresponding to each key in the supplied
524 hierarchy. Only keys that exist in the hierarchy will have
525 a value.
526 complete : `bool`
527 `True` if the full hierarchy exists and the final element
528 in ``hierarchy`` is the value of relevant value.
529 """
530 d = self._data
532 def checkNextItem(k, d, create):
533 """See if k is in d and if it is return the new child"""
534 nextVal = None
535 isThere = False
536 if d is None: 536 ↛ 538line 536 didn't jump to line 538, because the condition on line 536 was never true
537 # We have gone past the end of the hierarchy
538 pass
539 elif isinstance(d, collections.abc.Sequence): 539 ↛ 544line 539 didn't jump to line 544, because the condition on line 539 was never true
540 # Check sequence first because for lists
541 # __contains__ checks whether value is found in list
542 # not whether the index exists in list. When we traverse
543 # the hierarchy we are interested in the index.
544 try:
545 nextVal = d[int(k)]
546 isThere = True
547 except IndexError:
548 pass
549 except ValueError:
550 isThere = k in d
551 elif k in d:
552 nextVal = d[k]
553 isThere = True
554 elif create: 554 ↛ 555line 554 didn't jump to line 555, because the condition on line 554 was never true
555 d[k] = {}
556 nextVal = d[k]
557 isThere = True
558 return nextVal, isThere
560 hierarchy = []
561 complete = True
562 for k in keys:
563 d, isThere = checkNextItem(k, d, create)
564 if isThere:
565 hierarchy.append(d)
566 else:
567 complete = False
568 break
570 return hierarchy, complete
572 def __getitem__(self, name):
573 # Override the split for the simple case where there is an exact
574 # match. This allows `Config.items()` to work via a simple
575 # __iter__ implementation that returns top level keys of
576 # self._data.
577 keys = self._getKeyHierarchy(name)
579 hierarchy, complete = self._findInHierarchy(keys)
580 if not complete: 580 ↛ 581line 580 didn't jump to line 581, because the condition on line 580 was never true
581 raise KeyError(f"{name} not found")
582 data = hierarchy[-1]
584 if isinstance(data, collections.abc.Mapping):
585 data = Config(data)
586 # Ensure that child configs inherit the parent internal delimiter
587 if self._D != Config._D: 587 ↛ 588line 587 didn't jump to line 588, because the condition on line 587 was never true
588 data._D = self._D
589 return data
591 def __setitem__(self, name, value):
592 keys = self._getKeyHierarchy(name)
593 last = keys.pop()
594 if isinstance(value, Config):
595 value = copy.deepcopy(value._data)
597 hierarchy, complete = self._findInHierarchy(keys, create=True)
598 if hierarchy:
599 data = hierarchy[-1]
600 else:
601 data = self._data
603 try:
604 data[last] = value
605 except TypeError:
606 data[int(last)] = value
608 def __contains__(self, key):
609 keys = self._getKeyHierarchy(key)
610 hierarchy, complete = self._findInHierarchy(keys)
611 return complete
613 def __delitem__(self, key):
614 keys = self._getKeyHierarchy(key)
615 last = keys.pop()
616 hierarchy, complete = self._findInHierarchy(keys)
617 if complete: 617 ↛ 624line 617 didn't jump to line 624, because the condition on line 617 was never false
618 if hierarchy: 618 ↛ 619line 618 didn't jump to line 619, because the condition on line 618 was never true
619 data = hierarchy[-1]
620 else:
621 data = self._data
622 del data[last]
623 else:
624 raise KeyError(f"{key} not found in Config")
626 def update(self, other):
627 """Like dict.update, but will add or modify keys in nested dicts,
628 instead of overwriting the nested dict entirely.
630 For example, for the given code:
631 foo = {"a": {"b": 1}}
632 foo.update({"a": {"c": 2}})
634 Parameters
635 ----------
636 other : `dict` or `Config`
637 Source of configuration:
639 - If foo is a dict, then after the update foo == {"a": {"c": 2}}
640 - But if foo is a Config, then after the update
641 foo == {"a": {"b": 1, "c": 2}}
642 """
643 def doUpdate(d, u):
644 if not isinstance(u, collections.abc.Mapping) or \ 644 ↛ 646line 644 didn't jump to line 646, because the condition on line 644 was never true
645 not isinstance(d, collections.abc.Mapping):
646 raise RuntimeError("Only call update with Mapping, not {}".format(type(d)))
647 for k, v in u.items():
648 if isinstance(v, collections.abc.Mapping):
649 d[k] = doUpdate(d.get(k, {}), v)
650 else:
651 d[k] = v
652 return d
653 doUpdate(self._data, other)
655 def merge(self, other):
656 """Like Config.update, but will add keys & values from other that
657 DO NOT EXIST in self.
659 Keys and values that already exist in self will NOT be overwritten.
661 Parameters
662 ----------
663 other : `dict` or `Config`
664 Source of configuration:
665 """
666 otherCopy = copy.deepcopy(other)
667 otherCopy.update(self)
668 self._data = otherCopy._data
670 def nameTuples(self, topLevelOnly=False):
671 """Get tuples representing the name hierarchies of all keys.
673 The tuples returned from this method are guaranteed to be usable
674 to access items in the configuration object.
676 Parameters
677 ----------
678 topLevelOnly : `bool`, optional
679 If False, the default, a full hierarchy of names is returned.
680 If True, only the top level are returned.
682 Returns
683 -------
684 names : `list` of `tuple` of `str`
685 List of all names present in the `Config` where each element
686 in the list is a `tuple` of strings representing the hierarchy.
687 """
688 if topLevelOnly: 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true
689 return list((k,) for k in self)
691 def getKeysAsTuples(d, keys, base):
692 if isinstance(d, collections.abc.Sequence):
693 theseKeys = range(len(d))
694 else:
695 theseKeys = d.keys()
696 for key in theseKeys:
697 val = d[key]
698 levelKey = base + (key,) if base is not None else (key,)
699 keys.append(levelKey)
700 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \
701 and not isinstance(val, str):
702 getKeysAsTuples(val, keys, levelKey)
703 keys = []
704 getKeysAsTuples(self._data, keys, None)
705 return keys
707 def names(self, topLevelOnly=False, delimiter=None):
708 """Get a delimited name of all the keys in the hierarchy.
710 The values returned from this method are guaranteed to be usable
711 to access items in the configuration object.
713 Parameters
714 ----------
715 topLevelOnly : `bool`, optional
716 If False, the default, a full hierarchy of names is returned.
717 If True, only the top level are returned.
718 delimiter : `str`, optional
719 Delimiter to use when forming the keys. If the delimiter is
720 present in any of the keys, it will be escaped in the returned
721 names. If `None` given a delimiter will be automatically provided.
722 The delimiter can not be alphanumeric.
724 Returns
725 -------
726 names : `list` of `str`
727 List of all names present in the `Config`.
729 Notes
730 -----
731 This is different than the built-in method `dict.keys`, which will
732 return only the first level keys.
734 Raises
735 ------
736 ValueError:
737 The supplied delimiter is alphanumeric.
738 """
739 if topLevelOnly:
740 return list(self.keys())
742 # Get all the tuples of hierarchical keys
743 nameTuples = self.nameTuples()
745 if delimiter is not None and delimiter.isalnum():
746 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.")
748 if delimiter is None:
749 # Start with something, and ensure it does not need to be
750 # escaped (it is much easier to understand if not escaped)
751 delimiter = self._D
753 # Form big string for easy check of delimiter clash
754 combined = "".join("".join(str(s) for s in k) for k in nameTuples)
756 # Try a delimiter and keep trying until we get something that
757 # works.
758 ntries = 0
759 while delimiter in combined:
760 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.")
761 ntries += 1
763 if ntries > 100:
764 raise ValueError(f"Unable to determine a delimiter for Config {self}")
766 # try another one
767 while True:
768 delimiter = chr(ord(delimiter)+1)
769 if not delimiter.isalnum():
770 break
772 log.debug(f"Using delimiter {delimiter!r}")
774 # Form the keys, escaping the delimiter if necessary
775 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k)
776 for k in nameTuples]
777 return strings
779 def asArray(self, name):
780 """Get a value as an array.
782 May contain one or more elements.
784 Parameters
785 ----------
786 name : `str`
787 Key to use to retrieve value.
789 Returns
790 -------
791 array : `collections.abc.Sequence`
792 The value corresponding to name, but guaranteed to be returned
793 as a list with at least one element. If the value is a
794 `~collections.abc.Sequence` (and not a `str`) the value itself
795 will be returned, else the value will be the first element.
796 """
797 val = self.get(name)
798 if isinstance(val, str):
799 val = [val]
800 elif not isinstance(val, collections.abc.Sequence):
801 val = [val]
802 return val
804 def __eq__(self, other):
805 if isinstance(other, Config):
806 other = other._data
807 return self._data == other
809 def __ne__(self, other):
810 if isinstance(other, Config):
811 other = other._data
812 return self._data != other
814 #######
815 # i/o #
817 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]:
818 """Writes the config to an output stream.
820 Parameters
821 ----------
822 output : `IO`, optional
823 The stream to use for output. If `None` the serialized content
824 will be returned.
825 format : `str`, optional
826 The format to use for the output. Can be "yaml" or "json".
828 Returns
829 -------
830 serialized : `str` or `None`
831 If a stream was given the stream will be used and the return
832 value will be `None`. If the stream was `None` the
833 serialization will be returned as a string.
834 """
835 if format == "yaml":
836 return yaml.safe_dump(self._data, output, default_flow_style=False)
837 elif format == "json":
838 if output is not None:
839 json.dump(self._data, output, ensure_ascii=False)
840 return
841 else:
842 return json.dumps(self._data, ensure_ascii=False)
843 raise ValueError(f"Unsupported format for Config serialization: {format}")
845 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
846 defaultFileName: str = "butler.yaml",
847 overwrite: bool = True) -> None:
848 """Writes the config to location pointed to by given URI.
850 Currently supports 's3' and 'file' URI schemes.
852 Parameters
853 ----------
854 uri: `str` or `ButlerURI`
855 URI of location where the Config will be written.
856 updateFile : bool, optional
857 If True and uri does not end on a filename with extension, will
858 append `defaultFileName` to the target uri. True by default.
859 defaultFileName : bool, optional
860 The file name that will be appended to target uri if updateFile is
861 True and uri does not end on a file with an extension.
862 overwrite : bool, optional
863 If True the configuration will be written even if it already
864 exists at that location.
865 """
866 # Make local copy of URI or create new one
867 uri = ButlerURI(uri)
869 if updateFile and not uri.getExtension():
870 uri.updateFile(defaultFileName)
872 # Try to work out the format from the extension
873 ext = uri.getExtension()
874 format = ext[1:].lower()
876 uri.write(self.dump(format=format).encode(), overwrite=overwrite)
877 self.configFile = uri
879 @staticmethod
880 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True):
881 """Generic helper function for updating specific config parameters.
883 Allows for named parameters to be set to new values in bulk, and
884 for other values to be set by copying from a reference config.
886 Assumes that the supplied config is compatible with ``configType``
887 and will attach the updated values to the supplied config by
888 looking for the related component key. It is assumed that
889 ``config`` and ``full`` are from the same part of the
890 configuration hierarchy.
892 Parameters
893 ----------
894 configType : `ConfigSubset`
895 Config type to use to extract relevant items from ``config``.
896 config : `Config`
897 A `Config` to update. Only the subset understood by
898 the supplied `ConfigSubset` will be modified. Default values
899 will not be inserted and the content will not be validated
900 since mandatory keys are allowed to be missing until
901 populated later by merging.
902 full : `Config`
903 A complete config with all defaults expanded that can be
904 converted to a ``configType``. Read-only and will not be
905 modified by this method. Values are read from here if
906 ``toCopy`` is defined.
908 Repository-specific options that should not be obtained
909 from defaults when Butler instances are constructed
910 should be copied from ``full`` to ``config``.
911 toUpdate : `dict`, optional
912 A `dict` defining the keys to update and the new value to use.
913 The keys and values can be any supported by `Config`
914 assignment.
915 toCopy : `tuple`, optional
916 `tuple` of keys whose values should be copied from ``full``
917 into ``config``.
918 overwrite : `bool`, optional
919 If `False`, do not modify a value in ``config`` if the key
920 already exists. Default is always to overwrite.
922 Raises
923 ------
924 ValueError
925 Neither ``toUpdate`` not ``toCopy`` were defined.
926 """
927 if toUpdate is None and toCopy is None:
928 raise ValueError("One of toUpdate or toCopy parameters must be set.")
930 # If this is a parent configuration then we need to ensure that
931 # the supplied config has the relevant component key in it.
932 # If this is a parent configuration we add in the stub entry
933 # so that the ConfigSubset constructor will do the right thing.
934 # We check full for this since that is guaranteed to be complete.
935 if configType.component in full and configType.component not in config:
936 config[configType.component] = {}
938 # Extract the part of the config we wish to update
939 localConfig = configType(config, mergeDefaults=False, validate=False)
941 if toUpdate:
942 for key, value in toUpdate.items():
943 if key in localConfig and not overwrite:
944 log.debug("Not overriding key '%s' with value '%s' in config %s",
945 key, value, localConfig.__class__.__name__)
946 else:
947 localConfig[key] = value
949 if toCopy:
950 localFullConfig = configType(full, mergeDefaults=False)
951 for key in toCopy:
952 if key in localConfig and not overwrite:
953 log.debug("Not overriding key '%s' from defaults in config %s",
954 key, localConfig.__class__.__name__)
955 else:
956 localConfig[key] = localFullConfig[key]
958 # Reattach to parent if this is a child config
959 if configType.component in config:
960 config[configType.component] = localConfig
961 else:
962 config.update(localConfig)
964 def toDict(self):
965 """Convert a `Config` to a standalone hierarchical `dict`.
967 Returns
968 -------
969 d : `dict`
970 The standalone hierarchical `dict` with any `Config` classes
971 in the hierarchy converted to `dict`.
973 Notes
974 -----
975 This can be useful when passing a Config to some code that
976 expects native Python types.
977 """
978 output = copy.deepcopy(self._data)
979 for k, v in output.items():
980 if isinstance(v, Config):
981 v = v.toDict()
982 output[k] = v
983 return output
986class ConfigSubset(Config):
987 """Config representing a subset of a more general configuration.
989 Subclasses define their own component and when given a configuration
990 that includes that component, the resulting configuration only includes
991 the subset. For example, your config might contain ``dimensions`` if it's
992 part of a global config and that subset will be stored. If ``dimensions``
993 can not be found it is assumed that the entire contents of the
994 configuration should be used.
996 Default values are read from the environment or supplied search paths
997 using the default configuration file name specified in the subclass.
998 This allows a configuration class to be instantiated without any
999 additional arguments.
1001 Additional validation can be specified to check for keys that are mandatory
1002 in the configuration.
1004 Parameters
1005 ----------
1006 other : `Config` or `str` or `dict`
1007 Argument specifying the configuration information as understood
1008 by `Config`
1009 validate : `bool`, optional
1010 If `True` required keys will be checked to ensure configuration
1011 consistency.
1012 mergeDefaults : `bool`, optional
1013 If `True` defaults will be read and the supplied config will
1014 be combined with the defaults, with the supplied valiues taking
1015 precedence.
1016 searchPaths : `list` or `tuple`, optional
1017 Explicit additional paths to search for defaults. They should
1018 be supplied in priority order. These paths have higher priority
1019 than those read from the environment in
1020 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
1021 the local file system or URIs, `ButlerURI`.
1022 """
1024 component: ClassVar[Optional[str]] = None
1025 """Component to use from supplied config. Can be None. If specified the
1026 key is not required. Can be a full dot-separated path to a component.
1027 """
1029 requiredKeys: ClassVar[Sequence[str]] = ()
1030 """Keys that are required to be specified in the configuration.
1031 """
1033 defaultConfigFile: ClassVar[Optional[str]] = None
1034 """Name of the file containing defaults for this config class.
1035 """
1037 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None):
1039 # Create a blank object to receive the defaults
1040 # Once we have the defaults we then update with the external values
1041 super().__init__()
1043 # Create a standard Config rather than subset
1044 externalConfig = Config(other)
1046 # Select the part we need from it
1047 # To simplify the use of !include we also check for the existence of
1048 # component.component (since the included files can themselves
1049 # include the component name)
1050 if self.component is not None: 1050 ↛ 1059line 1050 didn't jump to line 1059, because the condition on line 1050 was never false
1051 doubled = (self.component, self.component)
1052 # Must check for double depth first
1053 if doubled in externalConfig: 1053 ↛ 1054line 1053 didn't jump to line 1054, because the condition on line 1053 was never true
1054 externalConfig = externalConfig[doubled]
1055 elif self.component in externalConfig:
1056 externalConfig._data = externalConfig._data[self.component]
1058 # Default files read to create this configuration
1059 self.filesRead = []
1061 # Assume we are not looking up child configurations
1062 containerKey = None
1064 # Sometimes we do not want to merge with defaults.
1065 if mergeDefaults:
1067 # Supplied search paths have highest priority
1068 fullSearchPath = []
1069 if searchPaths: 1069 ↛ 1070line 1069 didn't jump to line 1070, because the condition on line 1069 was never true
1070 fullSearchPath.extend(searchPaths)
1072 # Read default paths from enviroment
1073 fullSearchPath.extend(self.defaultSearchPaths())
1075 # There are two places to find defaults for this particular config
1076 # - The "defaultConfigFile" defined in the subclass
1077 # - The class specified in the "cls" element in the config.
1078 # Read cls after merging in case it changes.
1079 if self.defaultConfigFile is not None: 1079 ↛ 1084line 1079 didn't jump to line 1084, because the condition on line 1079 was never false
1080 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile)
1082 # Can have a class specification in the external config (priority)
1083 # or from the defaults.
1084 pytype = None
1085 if "cls" in externalConfig: 1085 ↛ 1086line 1085 didn't jump to line 1086, because the condition on line 1085 was never true
1086 pytype = externalConfig["cls"]
1087 elif "cls" in self: 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true
1088 pytype = self["cls"]
1090 if pytype is not None: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the condition on line 1090 was never true
1091 try:
1092 cls = doImport(pytype)
1093 except ImportError as e:
1094 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e
1095 defaultsFile = cls.defaultConfigFile
1096 if defaultsFile is not None:
1097 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile)
1099 # Get the container key in case we need it
1100 try:
1101 containerKey = cls.containerKey
1102 except AttributeError:
1103 pass
1105 # Now update this object with the external values so that the external
1106 # values always override the defaults
1107 self.update(externalConfig)
1109 # If this configuration has child configurations of the same
1110 # config class, we need to expand those defaults as well.
1112 if mergeDefaults and containerKey is not None and containerKey in self: 1112 ↛ 1113line 1112 didn't jump to line 1113, because the condition on line 1112 was never true
1113 for idx, subConfig in enumerate(self[containerKey]):
1114 self[containerKey, idx] = type(self)(other=subConfig, validate=validate,
1115 mergeDefaults=mergeDefaults,
1116 searchPaths=searchPaths)
1118 if validate:
1119 self.validate()
1121 @classmethod
1122 def defaultSearchPaths(cls):
1123 """Read the environment to determine search paths to use for global
1124 defaults.
1126 Global defaults, at lowest priority, are found in the ``config``
1127 directory of the butler source tree. Additional defaults can be
1128 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS``
1129 which is a PATH-like variable where paths at the front of the list
1130 have priority over those later.
1132 Returns
1133 -------
1134 paths : `list`
1135 Returns a list of paths to search. The returned order is in
1136 priority with the highest priority paths first. The butler config
1137 configuration resources will not be included here but will
1138 always be searched last.
1140 Notes
1141 -----
1142 The environment variable is split on the standard ``:`` path separator.
1143 This currently makes it incompatible with usage of URIs.
1144 """
1145 # We can pick up defaults from multiple search paths
1146 # We fill defaults by using the butler config path and then
1147 # the config path environment variable in reverse order.
1148 defaultsPaths = []
1150 if CONFIG_PATH in os.environ: 1150 ↛ 1151line 1150 didn't jump to line 1151, because the condition on line 1150 was never true
1151 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep)
1152 defaultsPaths.extend(externalPaths)
1154 # Add the package defaults as a resource
1155 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs",
1156 forceDirectory=True))
1157 return defaultsPaths
1159 def _updateWithConfigsFromPath(self, searchPaths, configFile):
1160 """Search the supplied paths, merging the configuration values
1162 The values read will override values currently stored in the object.
1163 Every file found in the path will be read, such that the earlier
1164 path entries have higher priority.
1166 Parameters
1167 ----------
1168 searchPaths : `list` of `ButlerURI`, `str`
1169 Paths to search for the supplied configFile. This path
1170 is the priority order, such that files read from the
1171 first path entry will be selected over those read from
1172 a later path. Can contain `str` referring to the local file
1173 system or a URI string.
1174 configFile : `ButlerURI`
1175 File to locate in path. If absolute path it will be read
1176 directly and the search path will not be used. Can be a URI
1177 to an explicit resource (which will ignore the search path)
1178 which is assumed to exist.
1179 """
1180 uri = ButlerURI(configFile)
1181 if uri.isabs() and uri.exists(): 1181 ↛ 1183line 1181 didn't jump to line 1183, because the condition on line 1181 was never true
1182 # Assume this resource exists
1183 self._updateWithOtherConfigFile(configFile)
1184 self.filesRead.append(configFile)
1185 else:
1186 # Reverse order so that high priority entries
1187 # update the object last.
1188 for pathDir in reversed(searchPaths):
1189 if isinstance(pathDir, (str, ButlerURI)): 1189 ↛ 1196line 1189 didn't jump to line 1196, because the condition on line 1189 was never false
1190 pathDir = ButlerURI(pathDir, forceDirectory=True)
1191 file = pathDir.join(configFile)
1192 if file.exists(): 1192 ↛ 1188line 1192 didn't jump to line 1188, because the condition on line 1192 was never false
1193 self.filesRead.append(file)
1194 self._updateWithOtherConfigFile(file)
1195 else:
1196 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}")
1198 def _updateWithOtherConfigFile(self, file):
1199 """Read in some defaults and update.
1201 Update the configuration by reading the supplied file as a config
1202 of this class, and merging such that these values override the
1203 current values. Contents of the external config are not validated.
1205 Parameters
1206 ----------
1207 file : `Config`, `str`, `ButlerURI`, or `dict`
1208 Entity that can be converted to a `ConfigSubset`.
1209 """
1210 # Use this class to read the defaults so that subsetting can happen
1211 # correctly.
1212 externalConfig = type(self)(file, validate=False, mergeDefaults=False)
1213 self.update(externalConfig)
1215 def validate(self):
1216 """Check that mandatory keys are present in this configuration.
1218 Ignored if ``requiredKeys`` is empty."""
1219 # Validation
1220 missing = [k for k in self.requiredKeys if k not in self._data]
1221 if missing: 1221 ↛ 1222line 1221 didn't jump to line 1222, because the condition on line 1221 was never true
1222 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")