Coverage for python/lsst/verify/yamlutils.py: 22%
33 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:00 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:00 -0700
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Utilities for working with YAML documents."""
23from collections import OrderedDict
24from copy import deepcopy
25import yaml
27__all__ = ['load_ordered_yaml', 'load_all_ordered_yaml']
30def load_ordered_yaml(stream, **kwargs):
31 """Load a YAML document from a stream as an `~collections.OrderedDict`.
33 Parameters
34 ----------
35 stream :
36 A stream for a YAML file (made with `open` or `~io.StringIO`, for
37 example).
38 loader : optional
39 A YAML loader class. Default is ``yaml.Loader``.
40 object_pairs_hook : obj, optional
41 Class that YAML key-value pairs are loaded into by the ``loader``.
42 Default is `collections.OrderedDict`.
44 Returns
45 -------
46 yaml_doc : `~collections.OrderedDict`
47 The YAML document as an `~collections.OrderedDict` (or the type
48 specified in ``object_pairs_hook``).
49 """
50 OrderedLoader = _build_ordered_loader(**kwargs)
51 return yaml.load(stream, OrderedLoader)
54def load_all_ordered_yaml(stream, **kwargs):
55 r"""Load all YAML documents from a stream as a `list` of
56 `~collections.OrderedDict`\ s.
58 Parameters
59 ----------
60 stream :
61 A stream for a YAML file (made with `open` or `~io.StringIO`, for
62 example).
63 loader : optional
64 A YAML loader class. Default is ``yaml.Loader``.
65 object_pairs_hook : obj, optional
66 Class that YAML key-value pairs are loaded into by the ``loader``.
67 Default is `collections.OrderedDict`.
69 Returns
70 -------
71 yaml_docs : `list` of `~collections.OrderedDict`
72 The YAML documents as a `list` of `~collections.OrderedDict`\ s (or
73 the type specified in ``object_pairs_hook``).
74 """
75 OrderedLoader = _build_ordered_loader(**kwargs)
76 return yaml.load_all(stream, OrderedLoader)
79def _build_ordered_loader(Loader=yaml.CSafeLoader,
80 object_pairs_hook=OrderedDict):
81 # Solution from http://stackoverflow.com/a/21912744
83 class OrderedLoader(Loader):
84 pass
86 def construct_mapping(loader, node):
87 loader.flatten_mapping(node)
88 return object_pairs_hook(loader.construct_pairs(node))
90 OrderedLoader.add_constructor(
91 yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
92 construct_mapping)
94 return OrderedLoader
97def merge_documents(base_doc, new_doc):
98 r"""Merge the content of a dict-like object onto a base dict-like object,
99 recursively following embedded dictionaries and lists.
101 Parameters
102 ----------
103 base_doc : `dict`-like
104 The base document.
105 new_doc : `dict`-like
106 The new document. Content from the new document are added to the
107 base document. Matching keys from the new document will override
108 the base document and new keys in the document are added to the
109 base document.
111 Returns
112 -------
113 merged_doc : `~collections.OrderedDict`
114 The merged document. The contents of ``merged_doc`` are copies
115 of originals in ``base_doc`` and ``new_doc``.
117 Notes
118 -----
119 This function implements a key-value document merging algorithm
120 design for specification YAML documents. The rules are:
122 - Key-values from ``base_doc`` not present in ``new_doc`` are carried into
123 ``merged_doc``.
125 - Key-values from ``new_doc`` not present in ``base_doc`` are carried into
126 ``merged_doc``.
128 - If both ``new_doc`` and ``base_doc`` share a key and the value from
129 **either** is a scalar (not a `dict` or `list`-type), the value from
130 ``new_doc`` is carried into ``merged_doc``.
132 - If both ``new_doc`` and ``base_doc`` share a key and the value from
133 **both** is a sequence (`list`-type) then the list items from the
134 ``new_doc`` are **appended** to the ``base_doc``\ 's list.
136 - If both ``new_doc`` and ``base_doc`` share a key and the value from
137 **both** is a mapping (`dict`-type) then the two values are
138 merged by recursively calling this ``merge_documents`` function.
139 """
140 # Create a copy so that the base doc is not mutated
141 merged_doc = deepcopy(base_doc)
143 for new_key, new_value in new_doc.items():
144 if new_key in merged_doc:
145 # Deal with merge by created the 'merged_value' from base and new
146 base_value = merged_doc[new_key]
148 if isinstance(base_value, dict) and isinstance(new_value, dict):
149 # Recursively merge these two dictionaries
150 merged_value = merge_documents(base_value, new_value)
152 elif isinstance(base_value, list) and isinstance(new_value, list):
153 # Both are lists: merge by appending the new items to the end
154 # of the base items.
155 # Copy the base's list so we're not modify the input
156 merged_value = deepcopy(base_value)
157 merged_value.extend(deepcopy(new_value)) # modifies in-place
159 else:
160 # A scalar: just over-write the existing base value
161 merged_value = deepcopy(new_value)
163 # Done merging this key-value pair
164 merged_doc[new_key] = merged_value
166 else:
167 # Add the new key that isn't over-writing merged_doc
168 merged_doc[new_key] = deepcopy(new_value)
170 return merged_doc