Coverage for python/lsst/daf/butler/core/config.py: 44%

482 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-19 01:58 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import io 

31import json 

32import logging 

33import os 

34import pprint 

35import sys 

36from pathlib import Path 

37from typing import IO, Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Union 

38 

39import yaml 

40from lsst.resources import ResourcePath, ResourcePathExpression 

41from lsst.utils import doImport 

42from yaml.representer import Representer 

43 

44yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

45 

46 

47# Config module logger 

48log = logging.getLogger(__name__) 

49 

50# PATH-like environment variable to use for defaults. 

51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

52 

53try: 

54 yamlLoader = yaml.CSafeLoader 

55except AttributeError: 

56 # Not all installations have the C library 

57 # (but assume for mypy's sake that they're the same) 

58 yamlLoader = yaml.SafeLoader # type: ignore 

59 

60 

61def _doUpdate(d, u): 

62 if not isinstance(u, collections.abc.Mapping) or not isinstance(d, collections.abc.MutableMapping): 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true

63 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

64 for k, v in u.items(): 

65 if isinstance(v, collections.abc.Mapping): 

66 d[k] = _doUpdate(d.get(k, {}), v) 

67 else: 

68 d[k] = v 

69 return d 

70 

71 

72def _checkNextItem(k, d, create, must_be_dict): 

73 """See if k is in d and if it is return the new child.""" 

74 nextVal = None 

75 isThere = False 

76 if d is None: 76 ↛ 78line 76 didn't jump to line 78, because the condition on line 76 was never true

77 # We have gone past the end of the hierarchy 

78 pass 

79 elif not must_be_dict and isinstance(d, collections.abc.Sequence): 79 ↛ 84line 79 didn't jump to line 84, because the condition on line 79 was never true

80 # Check for Sequence first because for lists 

81 # __contains__ checks whether value is found in list 

82 # not whether the index exists in list. When we traverse 

83 # the hierarchy we are interested in the index. 

84 try: 

85 nextVal = d[int(k)] 

86 isThere = True 

87 except IndexError: 

88 pass 

89 except ValueError: 

90 isThere = k in d 

91 elif k in d: 

92 nextVal = d[k] 

93 isThere = True 

94 elif create: 94 ↛ 95line 94 didn't jump to line 95, because the condition on line 94 was never true

95 d[k] = {} 

96 nextVal = d[k] 

97 isThere = True 

98 

99 return nextVal, isThere 

100 

101 

102class Loader(yamlLoader): 

103 """YAML Loader that supports file include directives. 

104 

105 Uses ``!include`` directive in a YAML file to point to another 

106 YAML file to be included. The path in the include directive is relative 

107 to the file containing that directive. 

108 

109 storageClasses: !include storageClasses.yaml 

110 

111 Examples 

112 -------- 

113 >>> with open("document.yaml", "r") as f: 

114 data = yaml.load(f, Loader=Loader) 

115 

116 Notes 

117 ----- 

118 See https://davidchall.github.io/yaml-includes.html 

119 """ 

120 

121 def __init__(self, stream): 

122 super().__init__(stream) 

123 # if this is a string and not a stream we may well lack a name 

124 try: 

125 self._root = ResourcePath(stream.name) 

126 except AttributeError: 

127 # No choice but to assume a local filesystem 

128 self._root = ResourcePath("no-file.yaml") 

129 Loader.add_constructor("!include", Loader.include) 

130 

131 def include(self, node): 

132 result: Union[List[Any], Dict[str, Any]] 

133 if isinstance(node, yaml.ScalarNode): 

134 return self.extractFile(self.construct_scalar(node)) 

135 

136 elif isinstance(node, yaml.SequenceNode): 

137 result = [] 

138 for filename in self.construct_sequence(node): 

139 result.append(self.extractFile(filename)) 

140 return result 

141 

142 elif isinstance(node, yaml.MappingNode): 

143 result = {} 

144 for k, v in self.construct_mapping(node).items(): 

145 result[k] = self.extractFile(v) 

146 return result 

147 

148 else: 

149 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

150 raise yaml.constructor.ConstructorError 

151 

152 def extractFile(self, filename): 

153 # It is possible for the !include to point to an explicit URI 

154 # instead of a relative URI, therefore we first see if it is 

155 # scheme-less or not. If it has a scheme we use it directly 

156 # if it is scheme-less we use it relative to the file root. 

157 requesteduri = ResourcePath(filename, forceAbsolute=False) 

158 

159 if requesteduri.scheme: 

160 fileuri = requesteduri 

161 else: 

162 fileuri = self._root.updatedFile(filename) 

163 

164 log.debug("Opening YAML file via !include: %s", fileuri) 

165 

166 # Read all the data from the resource 

167 data = fileuri.read() 

168 

169 # Store the bytes into a BytesIO so we can attach a .name 

170 stream = io.BytesIO(data) 

171 stream.name = fileuri.geturl() 

172 return yaml.load(stream, Loader) 

173 

174 

175class Config(collections.abc.MutableMapping): 

176 r"""Implements a datatype that is used by `Butler` for configuration. 

177 

178 It is essentially a `dict` with key/value pairs, including nested dicts 

179 (as values). In fact, it can be initialized with a `dict`. 

180 This is explained next: 

181 

182 Config extends the `dict` api so that hierarchical values may be accessed 

183 with delimited notation or as a tuple. If a string is given the delimiter 

184 is picked up from the first character in that string. For example, 

185 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

186 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

187 If the first character is alphanumeric, no delimiter will be used. 

188 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

189 Unicode characters can be used as the delimiter for distinctiveness if 

190 required. 

191 

192 If a key in the hierarchy starts with a non-alphanumeric character care 

193 should be used to ensure that either the tuple interface is used or 

194 a distinct delimiter is always given in string form. 

195 

196 Finally, the delimiter can be escaped if it is part of a key and also 

197 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

198 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

199 always better to use a different delimiter in these cases. 

200 

201 Note that adding a multi-level key implicitly creates any nesting levels 

202 that do not exist, but removing multi-level keys does not automatically 

203 remove empty nesting levels. As a result: 

204 

205 >>> c = Config() 

206 >>> c[".a.b"] = 1 

207 >>> del c[".a.b"] 

208 >>> c["a"] 

209 Config({'a': {}}) 

210 

211 Storage formats supported: 

212 

213 - yaml: read and write is supported. 

214 - json: read and write is supported but no ``!include`` directive. 

215 

216 Parameters 

217 ---------- 

218 other : `lsst.resources.ResourcePath` or `Config` or `dict` 

219 Other source of configuration, can be: 

220 

221 - (`lsst.resources.ResourcePathExpression`) 

222 Treated as a URI to a config file. Must end with ".yaml". 

223 - (`Config`) Copies the other Config's values into this one. 

224 - (`dict`) Copies the values from the dict into this Config. 

225 

226 If `None` is provided an empty `Config` will be created. 

227 """ 

228 

229 _D: str = "→" 

230 """Default internal delimiter to use for components in the hierarchy when 

231 constructing keys for external use (see `Config.names()`).""" 

232 

233 includeKey: ClassVar[str] = "includeConfigs" 

234 """Key used to indicate that another config should be included at this 

235 part of the hierarchy.""" 

236 

237 resourcesPackage: str = "lsst.daf.butler" 

238 """Package to search for default configuration data. The resources 

239 themselves will be within a ``configs`` resource hierarchy.""" 

240 

241 def __init__(self, other=None): 

242 self._data: Dict[str, Any] = {} 

243 self.configFile = None 

244 

245 if other is None: 

246 return 

247 

248 if isinstance(other, Config): 

249 self._data = copy.deepcopy(other._data) 

250 self.configFile = other.configFile 

251 elif isinstance(other, (dict, collections.abc.Mapping)): 

252 # In most cases we have a dict, and it's more efficient 

253 # to check for a dict instance before checking the generic mapping. 

254 self.update(other) 

255 elif isinstance(other, (str, ResourcePath, Path)): 255 ↛ 262line 255 didn't jump to line 262, because the condition on line 255 was never false

256 # if other is a string, assume it is a file path/URI 

257 self.__initFromUri(other) 

258 self._processExplicitIncludes() 

259 else: 

260 # if the config specified by other could not be recognized raise 

261 # a runtime error. 

262 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

263 

264 def ppprint(self): 

265 """Return config as formatted readable string. 

266 

267 Examples 

268 -------- 

269 use: ``pdb> print(myConfigObject.ppprint())`` 

270 

271 Returns 

272 ------- 

273 s : `str` 

274 A prettyprint formatted string representing the config 

275 """ 

276 return pprint.pformat(self._data, indent=2, width=1) 

277 

278 def __repr__(self): 

279 return f"{type(self).__name__}({self._data!r})" 

280 

281 def __str__(self): 

282 return self.ppprint() 

283 

284 def __len__(self): 

285 return len(self._data) 

286 

287 def __iter__(self): 

288 return iter(self._data) 

289 

290 def copy(self): 

291 return type(self)(self) 

292 

293 @classmethod 

294 def fromString(cls, string: str, format: str = "yaml") -> Config: 

295 """Create a new Config instance from a serialized string. 

296 

297 Parameters 

298 ---------- 

299 string : `str` 

300 String containing content in specified format 

301 format : `str`, optional 

302 Format of the supplied string. Can be ``json`` or ``yaml``. 

303 

304 Returns 

305 ------- 

306 c : `Config` 

307 Newly-constructed Config. 

308 """ 

309 if format == "yaml": 

310 new_config = cls().__initFromYaml(string) 

311 elif format == "json": 

312 new_config = cls().__initFromJson(string) 

313 else: 

314 raise ValueError(f"Unexpected format of string: {format}") 

315 new_config._processExplicitIncludes() 

316 return new_config 

317 

318 @classmethod 

319 def fromYaml(cls, string: str) -> Config: 

320 """Create a new Config instance from a YAML string. 

321 

322 Parameters 

323 ---------- 

324 string : `str` 

325 String containing content in YAML format 

326 

327 Returns 

328 ------- 

329 c : `Config` 

330 Newly-constructed Config. 

331 """ 

332 return cls.fromString(string, format="yaml") 

333 

334 def __initFromUri(self, path: ResourcePathExpression) -> None: 

335 """Load a file from a path or an URI. 

336 

337 Parameters 

338 ---------- 

339 path : `lsst.resources.ResourcePathExpression` 

340 Path or a URI to a persisted config file. 

341 """ 

342 uri = ResourcePath(path) 

343 ext = uri.getExtension() 

344 if ext == ".yaml": 344 ↛ 351line 344 didn't jump to line 351, because the condition on line 344 was never false

345 log.debug("Opening YAML config file: %s", uri.geturl()) 

346 content = uri.read() 

347 # Use a stream so we can name it 

348 stream = io.BytesIO(content) 

349 stream.name = uri.geturl() 

350 self.__initFromYaml(stream) 

351 elif ext == ".json": 

352 log.debug("Opening JSON config file: %s", uri.geturl()) 

353 content = uri.read() 

354 self.__initFromJson(content) 

355 else: 

356 # This URI does not have a valid extension. It might be because 

357 # we ended up with a directory and not a file. Before we complain 

358 # about an extension, do an existence check. No need to do 

359 # the (possibly expensive) existence check in the default code 

360 # path above because we will find out soon enough that the file 

361 # is not there. 

362 if not uri.exists(): 

363 raise FileNotFoundError(f"Config location {uri} does not exist.") 

364 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

365 self.configFile = uri 

366 

367 def __initFromYaml(self, stream): 

368 """Load a YAML config from any readable stream that contains one. 

369 

370 Parameters 

371 ---------- 

372 stream: `IO` or `str` 

373 Stream to pass to the YAML loader. Accepts anything that 

374 `yaml.load` accepts. This can include a string as well as an 

375 IO stream. 

376 

377 Raises 

378 ------ 

379 yaml.YAMLError 

380 If there is an error loading the file. 

381 """ 

382 content = yaml.load(stream, Loader=Loader) 

383 if content is None: 383 ↛ 384line 383 didn't jump to line 384, because the condition on line 383 was never true

384 content = {} 

385 self._data = content 

386 return self 

387 

388 def __initFromJson(self, stream): 

389 """Load a JSON config from any readable stream that contains one. 

390 

391 Parameters 

392 ---------- 

393 stream: `IO` or `str` 

394 Stream to pass to the JSON loader. This can include a string as 

395 well as an IO stream. 

396 

397 Raises 

398 ------ 

399 TypeError: 

400 Raised if there is an error loading the content. 

401 """ 

402 if isinstance(stream, (bytes, str)): 

403 content = json.loads(stream) 

404 else: 

405 content = json.load(stream) 

406 if content is None: 

407 content = {} 

408 self._data = content 

409 return self 

410 

411 def _processExplicitIncludes(self): 

412 """Scan through the configuration searching for the special includes. 

413 

414 Looks for ``includeConfigs`` directive and processes the includes. 

415 """ 

416 # Search paths for config files 

417 searchPaths = [ResourcePath(os.path.curdir, forceDirectory=True)] 

418 if self.configFile is not None: 418 ↛ 426line 418 didn't jump to line 426, because the condition on line 418 was never false

419 if isinstance(self.configFile, ResourcePath): 419 ↛ 422line 419 didn't jump to line 422, because the condition on line 419 was never false

420 configDir = self.configFile.dirname() 

421 else: 

422 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

423 searchPaths.append(configDir) 

424 

425 # Ensure we know what delimiter to use 

426 names = self.nameTuples() 

427 for path in names: 

428 if path[-1] == self.includeKey: 428 ↛ 430line 428 didn't jump to line 430, because the condition on line 428 was never true

429 

430 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

431 basePath = path[:-1] 

432 

433 # Extract the includes and then delete them from the config 

434 includes = self[path] 

435 del self[path] 

436 

437 # Be consistent and convert to a list 

438 if not isinstance(includes, list): 

439 includes = [includes] 

440 

441 # Read each file assuming it is a reference to a file 

442 # The file can be relative to config file or cwd 

443 # ConfigSubset search paths are not used 

444 subConfigs = [] 

445 for fileName in includes: 

446 # Expand any shell variables -- this could be URI 

447 fileName = ResourcePath(os.path.expandvars(fileName), forceAbsolute=False) 

448 found = None 

449 if fileName.isabs(): 

450 found = fileName 

451 else: 

452 for dir in searchPaths: 

453 if isinstance(dir, ResourcePath): 

454 specific = dir.join(fileName.path) 

455 # Remote resource check might be expensive 

456 if specific.exists(): 

457 found = specific 

458 else: 

459 log.warning( 

460 "Do not understand search path entry '%s' of type %s", 

461 dir, 

462 type(dir).__name__, 

463 ) 

464 if not found: 

465 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

466 

467 # Read the referenced Config as a Config 

468 subConfigs.append(type(self)(found)) 

469 

470 # Now we need to merge these sub configs with the current 

471 # information that was present in this node in the config 

472 # tree with precedence given to the explicit values 

473 newConfig = subConfigs.pop(0) 

474 for sc in subConfigs: 

475 newConfig.update(sc) 

476 

477 # Explicit values take precedence 

478 if not basePath: 

479 # This is an include at the root config 

480 newConfig.update(self) 

481 # Replace the current config 

482 self._data = newConfig._data 

483 else: 

484 newConfig.update(self[basePath]) 

485 # And reattach to the base config 

486 self[basePath] = newConfig 

487 

488 @staticmethod 

489 def _splitIntoKeys(key): 

490 r"""Split the argument for get/set/in into a hierarchical list. 

491 

492 Parameters 

493 ---------- 

494 key : `str` or iterable 

495 Argument given to get/set/in. If an iterable is provided it will 

496 be converted to a list. If the first character of the string 

497 is not an alphanumeric character then it will be used as the 

498 delimiter for the purposes of splitting the remainder of the 

499 string. If the delimiter is also in one of the keys then it 

500 can be escaped using ``\``. There is no default delimiter. 

501 

502 Returns 

503 ------- 

504 keys : `list` 

505 Hierarchical keys as a `list`. 

506 """ 

507 if isinstance(key, str): 

508 if not key[0].isalnum(): 508 ↛ 509line 508 didn't jump to line 509, because the condition on line 508 was never true

509 d = key[0] 

510 key = key[1:] 

511 else: 

512 return [ 

513 key, 

514 ] 

515 escaped = f"\\{d}" 

516 temp = None 

517 if escaped in key: 

518 # Complain at the attempt to escape the escape 

519 doubled = rf"\{escaped}" 

520 if doubled in key: 

521 raise ValueError( 

522 f"Escaping an escaped delimiter ({doubled} in {key}) is not yet supported." 

523 ) 

524 # Replace with a character that won't be in the string 

525 temp = "\r" 

526 if temp in key or d == temp: 

527 raise ValueError( 

528 f"Can not use character {temp!r} in hierarchical key or as" 

529 " delimiter if escaping the delimiter" 

530 ) 

531 key = key.replace(escaped, temp) 

532 hierarchy = key.split(d) 

533 if temp: 

534 hierarchy = [h.replace(temp, d) for h in hierarchy] 

535 return hierarchy 

536 elif isinstance(key, collections.abc.Iterable): 536 ↛ 540line 536 didn't jump to line 540, because the condition on line 536 was never false

537 return list(key) 

538 else: 

539 # Not sure what this is so try it anyway 

540 return [ 

541 key, 

542 ] 

543 

544 def _getKeyHierarchy(self, name): 

545 """Retrieve the key hierarchy for accessing the Config. 

546 

547 Parameters 

548 ---------- 

549 name : `str` or `tuple` 

550 Delimited string or `tuple` of hierarchical keys. 

551 

552 Returns 

553 ------- 

554 hierarchy : `list` of `str` 

555 Hierarchy to use as a `list`. If the name is available directly 

556 as a key in the Config it will be used regardless of the presence 

557 of any nominal delimiter. 

558 """ 

559 if name in self._data: 

560 keys = [ 

561 name, 

562 ] 

563 else: 

564 keys = self._splitIntoKeys(name) 

565 return keys 

566 

567 def _findInHierarchy(self, keys, create=False): 

568 """Look for hierarchy of keys in Config. 

569 

570 Parameters 

571 ---------- 

572 keys : `list` or `tuple` 

573 Keys to search in hierarchy. 

574 create : `bool`, optional 

575 If `True`, if a part of the hierarchy does not exist, insert an 

576 empty `dict` into the hierarchy. 

577 

578 Returns 

579 ------- 

580 hierarchy : `list` 

581 List of the value corresponding to each key in the supplied 

582 hierarchy. Only keys that exist in the hierarchy will have 

583 a value. 

584 complete : `bool` 

585 `True` if the full hierarchy exists and the final element 

586 in ``hierarchy`` is the value of relevant value. 

587 """ 

588 d = self._data 

589 

590 # For the first key, d must be a dict so it is a waste 

591 # of time to check for a sequence. 

592 must_be_dict = True 

593 

594 hierarchy = [] 

595 complete = True 

596 for k in keys: 

597 d, isThere = _checkNextItem(k, d, create, must_be_dict) 

598 if isThere: 

599 hierarchy.append(d) 

600 else: 

601 complete = False 

602 break 

603 # Second time round it might be a sequence. 

604 must_be_dict = False 

605 

606 return hierarchy, complete 

607 

608 def __getitem__(self, name): 

609 # Override the split for the simple case where there is an exact 

610 # match. This allows `Config.items()` to work via a simple 

611 # __iter__ implementation that returns top level keys of 

612 # self._data. 

613 

614 # If the name matches a key in the top-level hierarchy, bypass 

615 # all further cleverness. 

616 found_directly = False 

617 try: 

618 data = self._data[name] 

619 found_directly = True 

620 except KeyError: 

621 pass 

622 

623 if not found_directly: 623 ↛ 624line 623 didn't jump to line 624, because the condition on line 623 was never true

624 keys = self._getKeyHierarchy(name) 

625 

626 hierarchy, complete = self._findInHierarchy(keys) 

627 if not complete: 

628 raise KeyError(f"{name} not found") 

629 data = hierarchy[-1] 

630 

631 # In most cases we have a dict, and it's more efficient 

632 # to check for a dict instance before checking the generic mapping. 

633 if isinstance(data, (dict, collections.abc.Mapping)): 

634 data = Config(data) 

635 # Ensure that child configs inherit the parent internal delimiter 

636 if self._D != Config._D: 636 ↛ 637line 636 didn't jump to line 637, because the condition on line 636 was never true

637 data._D = self._D 

638 return data 

639 

640 def __setitem__(self, name, value): 

641 keys = self._getKeyHierarchy(name) 

642 last = keys.pop() 

643 if isinstance(value, Config): 

644 value = copy.deepcopy(value._data) 

645 

646 hierarchy, complete = self._findInHierarchy(keys, create=True) 

647 if hierarchy: 

648 data = hierarchy[-1] 

649 else: 

650 data = self._data 

651 

652 try: 

653 data[last] = value 

654 except TypeError: 

655 data[int(last)] = value 

656 

657 def __contains__(self, key): 

658 keys = self._getKeyHierarchy(key) 

659 hierarchy, complete = self._findInHierarchy(keys) 

660 return complete 

661 

662 def __delitem__(self, key): 

663 keys = self._getKeyHierarchy(key) 

664 last = keys.pop() 

665 hierarchy, complete = self._findInHierarchy(keys) 

666 if complete: 666 ↛ 673line 666 didn't jump to line 673, because the condition on line 666 was never false

667 if hierarchy: 667 ↛ 668line 667 didn't jump to line 668, because the condition on line 667 was never true

668 data = hierarchy[-1] 

669 else: 

670 data = self._data 

671 del data[last] 

672 else: 

673 raise KeyError(f"{key} not found in Config") 

674 

675 def update(self, other): 

676 """Update config from other `Config` or `dict`. 

677 

678 Like `dict.update()`, but will add or modify keys in nested dicts, 

679 instead of overwriting the nested dict entirely. 

680 

681 Parameters 

682 ---------- 

683 other : `dict` or `Config` 

684 Source of configuration: 

685 

686 Examples 

687 -------- 

688 >>> c = Config({"a": {"b": 1}}) 

689 >>> c.update({"a": {"c": 2}}) 

690 >>> print(c) 

691 {'a': {'b': 1, 'c': 2}} 

692 

693 >>> foo = {"a": {"b": 1}} 

694 >>> foo.update({"a": {"c": 2}}) 

695 >>> print(foo) 

696 {'a': {'c': 2}} 

697 """ 

698 _doUpdate(self._data, other) 

699 

700 def merge(self, other): 

701 """Merge another Config into this one. 

702 

703 Like `Config.update()`, but will add keys & values from other that 

704 DO NOT EXIST in self. 

705 

706 Keys and values that already exist in self will NOT be overwritten. 

707 

708 Parameters 

709 ---------- 

710 other : `dict` or `Config` 

711 Source of configuration: 

712 """ 

713 if not isinstance(other, collections.abc.Mapping): 

714 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}") 

715 

716 # Convert the supplied mapping to a Config for consistency 

717 # This will do a deepcopy if it is already a Config 

718 otherCopy = Config(other) 

719 otherCopy.update(self) 

720 self._data = otherCopy._data 

721 

722 def nameTuples(self, topLevelOnly=False): 

723 """Get tuples representing the name hierarchies of all keys. 

724 

725 The tuples returned from this method are guaranteed to be usable 

726 to access items in the configuration object. 

727 

728 Parameters 

729 ---------- 

730 topLevelOnly : `bool`, optional 

731 If False, the default, a full hierarchy of names is returned. 

732 If True, only the top level are returned. 

733 

734 Returns 

735 ------- 

736 names : `list` of `tuple` of `str` 

737 List of all names present in the `Config` where each element 

738 in the list is a `tuple` of strings representing the hierarchy. 

739 """ 

740 if topLevelOnly: 740 ↛ 741line 740 didn't jump to line 741, because the condition on line 740 was never true

741 return list((k,) for k in self) 

742 

743 def getKeysAsTuples(d, keys, base): 

744 if isinstance(d, collections.abc.Sequence): 

745 theseKeys = range(len(d)) 

746 else: 

747 theseKeys = d.keys() 

748 for key in theseKeys: 

749 val = d[key] 

750 levelKey = base + (key,) if base is not None else (key,) 

751 keys.append(levelKey) 

752 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) and not isinstance( 

753 val, str 

754 ): 

755 getKeysAsTuples(val, keys, levelKey) 

756 

757 keys: List[Tuple[str, ...]] = [] 

758 getKeysAsTuples(self._data, keys, None) 

759 return keys 

760 

761 def names(self, topLevelOnly=False, delimiter=None): 

762 """Get a delimited name of all the keys in the hierarchy. 

763 

764 The values returned from this method are guaranteed to be usable 

765 to access items in the configuration object. 

766 

767 Parameters 

768 ---------- 

769 topLevelOnly : `bool`, optional 

770 If False, the default, a full hierarchy of names is returned. 

771 If True, only the top level are returned. 

772 delimiter : `str`, optional 

773 Delimiter to use when forming the keys. If the delimiter is 

774 present in any of the keys, it will be escaped in the returned 

775 names. If `None` given a delimiter will be automatically provided. 

776 The delimiter can not be alphanumeric. 

777 

778 Returns 

779 ------- 

780 names : `list` of `str` 

781 List of all names present in the `Config`. 

782 

783 Notes 

784 ----- 

785 This is different than the built-in method `dict.keys`, which will 

786 return only the first level keys. 

787 

788 Raises 

789 ------ 

790 ValueError: 

791 The supplied delimiter is alphanumeric. 

792 """ 

793 if topLevelOnly: 

794 return list(self.keys()) 

795 

796 # Get all the tuples of hierarchical keys 

797 nameTuples = self.nameTuples() 

798 

799 if delimiter is not None and delimiter.isalnum(): 

800 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

801 

802 if delimiter is None: 

803 # Start with something, and ensure it does not need to be 

804 # escaped (it is much easier to understand if not escaped) 

805 delimiter = self._D 

806 

807 # Form big string for easy check of delimiter clash 

808 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

809 

810 # Try a delimiter and keep trying until we get something that 

811 # works. 

812 ntries = 0 

813 while delimiter in combined: 

814 log.debug("Delimiter '%s' could not be used. Trying another.", delimiter) 

815 ntries += 1 

816 

817 if ntries > 100: 

818 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

819 

820 # try another one 

821 while True: 

822 delimiter = chr(ord(delimiter) + 1) 

823 if not delimiter.isalnum(): 

824 break 

825 

826 log.debug("Using delimiter %r", delimiter) 

827 

828 # Form the keys, escaping the delimiter if necessary 

829 strings = [ 

830 delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

831 for k in nameTuples 

832 ] 

833 return strings 

834 

835 def asArray(self, name): 

836 """Get a value as an array. 

837 

838 May contain one or more elements. 

839 

840 Parameters 

841 ---------- 

842 name : `str` 

843 Key to use to retrieve value. 

844 

845 Returns 

846 ------- 

847 array : `collections.abc.Sequence` 

848 The value corresponding to name, but guaranteed to be returned 

849 as a list with at least one element. If the value is a 

850 `~collections.abc.Sequence` (and not a `str`) the value itself 

851 will be returned, else the value will be the first element. 

852 """ 

853 val = self.get(name) 

854 if isinstance(val, str): 

855 val = [val] 

856 elif not isinstance(val, collections.abc.Sequence): 

857 val = [val] 

858 return val 

859 

860 def __eq__(self, other): 

861 if isinstance(other, Config): 

862 other = other._data 

863 return self._data == other 

864 

865 def __ne__(self, other): 

866 if isinstance(other, Config): 

867 other = other._data 

868 return self._data != other 

869 

870 ####### 

871 # i/o # 

872 

873 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

874 """Write the config to an output stream. 

875 

876 Parameters 

877 ---------- 

878 output : `IO`, optional 

879 The stream to use for output. If `None` the serialized content 

880 will be returned. 

881 format : `str`, optional 

882 The format to use for the output. Can be "yaml" or "json". 

883 

884 Returns 

885 ------- 

886 serialized : `str` or `None` 

887 If a stream was given the stream will be used and the return 

888 value will be `None`. If the stream was `None` the 

889 serialization will be returned as a string. 

890 """ 

891 if format == "yaml": 

892 return yaml.safe_dump(self._data, output, default_flow_style=False) 

893 elif format == "json": 

894 if output is not None: 

895 json.dump(self._data, output, ensure_ascii=False) 

896 return None 

897 else: 

898 return json.dumps(self._data, ensure_ascii=False) 

899 raise ValueError(f"Unsupported format for Config serialization: {format}") 

900 

901 def dumpToUri( 

902 self, 

903 uri: ResourcePathExpression, 

904 updateFile: bool = True, 

905 defaultFileName: str = "butler.yaml", 

906 overwrite: bool = True, 

907 ) -> None: 

908 """Write the config to location pointed to by given URI. 

909 

910 Currently supports 's3' and 'file' URI schemes. 

911 

912 Parameters 

913 ---------- 

914 uri: `lsst.resources.ResourcePathExpression` 

915 URI of location where the Config will be written. 

916 updateFile : bool, optional 

917 If True and uri does not end on a filename with extension, will 

918 append `defaultFileName` to the target uri. True by default. 

919 defaultFileName : bool, optional 

920 The file name that will be appended to target uri if updateFile is 

921 True and uri does not end on a file with an extension. 

922 overwrite : bool, optional 

923 If True the configuration will be written even if it already 

924 exists at that location. 

925 """ 

926 # Make local copy of URI or create new one 

927 uri = ResourcePath(uri) 

928 

929 if updateFile and not uri.getExtension(): 

930 uri = uri.updatedFile(defaultFileName) 

931 

932 # Try to work out the format from the extension 

933 ext = uri.getExtension() 

934 format = ext[1:].lower() 

935 

936 output = self.dump(format=format) 

937 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

938 uri.write(output.encode(), overwrite=overwrite) 

939 self.configFile = uri 

940 

941 @staticmethod 

942 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True, toMerge=None): 

943 """Update specific config parameters. 

944 

945 Allows for named parameters to be set to new values in bulk, and 

946 for other values to be set by copying from a reference config. 

947 

948 Assumes that the supplied config is compatible with ``configType`` 

949 and will attach the updated values to the supplied config by 

950 looking for the related component key. It is assumed that 

951 ``config`` and ``full`` are from the same part of the 

952 configuration hierarchy. 

953 

954 Parameters 

955 ---------- 

956 configType : `ConfigSubset` 

957 Config type to use to extract relevant items from ``config``. 

958 config : `Config` 

959 A `Config` to update. Only the subset understood by 

960 the supplied `ConfigSubset` will be modified. Default values 

961 will not be inserted and the content will not be validated 

962 since mandatory keys are allowed to be missing until 

963 populated later by merging. 

964 full : `Config` 

965 A complete config with all defaults expanded that can be 

966 converted to a ``configType``. Read-only and will not be 

967 modified by this method. Values are read from here if 

968 ``toCopy`` is defined. 

969 

970 Repository-specific options that should not be obtained 

971 from defaults when Butler instances are constructed 

972 should be copied from ``full`` to ``config``. 

973 toUpdate : `dict`, optional 

974 A `dict` defining the keys to update and the new value to use. 

975 The keys and values can be any supported by `Config` 

976 assignment. 

977 toCopy : `tuple`, optional 

978 `tuple` of keys whose values should be copied from ``full`` 

979 into ``config``. 

980 overwrite : `bool`, optional 

981 If `False`, do not modify a value in ``config`` if the key 

982 already exists. Default is always to overwrite. 

983 toMerge : `tuple`, optional 

984 Keys to merge content from full to config without overwriting 

985 pre-existing values. Only works if the key refers to a hierarchy. 

986 The ``overwrite`` flag is ignored. 

987 

988 Raises 

989 ------ 

990 ValueError 

991 Neither ``toUpdate``, ``toCopy`` nor ``toMerge`` were defined. 

992 """ 

993 if toUpdate is None and toCopy is None and toMerge is None: 

994 raise ValueError("At least one of toUpdate, toCopy, or toMerge parameters must be set.") 

995 

996 # If this is a parent configuration then we need to ensure that 

997 # the supplied config has the relevant component key in it. 

998 # If this is a parent configuration we add in the stub entry 

999 # so that the ConfigSubset constructor will do the right thing. 

1000 # We check full for this since that is guaranteed to be complete. 

1001 if configType.component in full and configType.component not in config: 

1002 config[configType.component] = {} 

1003 

1004 # Extract the part of the config we wish to update 

1005 localConfig = configType(config, mergeDefaults=False, validate=False) 

1006 

1007 if toUpdate: 

1008 for key, value in toUpdate.items(): 

1009 if key in localConfig and not overwrite: 

1010 log.debug( 

1011 "Not overriding key '%s' with value '%s' in config %s", 

1012 key, 

1013 value, 

1014 localConfig.__class__.__name__, 

1015 ) 

1016 else: 

1017 localConfig[key] = value 

1018 

1019 if toCopy or toMerge: 

1020 localFullConfig = configType(full, mergeDefaults=False) 

1021 

1022 if toCopy: 

1023 for key in toCopy: 

1024 if key in localConfig and not overwrite: 

1025 log.debug( 

1026 "Not overriding key '%s' from defaults in config %s", 

1027 key, 

1028 localConfig.__class__.__name__, 

1029 ) 

1030 else: 

1031 localConfig[key] = localFullConfig[key] 

1032 if toMerge: 

1033 for key in toMerge: 

1034 if key in localConfig: 

1035 # Get the node from the config to do the merge 

1036 # but then have to reattach to the config. 

1037 subset = localConfig[key] 

1038 subset.merge(localFullConfig[key]) 

1039 localConfig[key] = subset 

1040 else: 

1041 localConfig[key] = localFullConfig[key] 

1042 

1043 # Reattach to parent if this is a child config 

1044 if configType.component in config: 

1045 config[configType.component] = localConfig 

1046 else: 

1047 config.update(localConfig) 

1048 

1049 def toDict(self): 

1050 """Convert a `Config` to a standalone hierarchical `dict`. 

1051 

1052 Returns 

1053 ------- 

1054 d : `dict` 

1055 The standalone hierarchical `dict` with any `Config` classes 

1056 in the hierarchy converted to `dict`. 

1057 

1058 Notes 

1059 ----- 

1060 This can be useful when passing a Config to some code that 

1061 expects native Python types. 

1062 """ 

1063 output = copy.deepcopy(self._data) 

1064 for k, v in output.items(): 

1065 if isinstance(v, Config): 1065 ↛ 1066line 1065 didn't jump to line 1066, because the condition on line 1065 was never true

1066 v = v.toDict() 

1067 output[k] = v 

1068 return output 

1069 

1070 

1071class ConfigSubset(Config): 

1072 """Config representing a subset of a more general configuration. 

1073 

1074 Subclasses define their own component and when given a configuration 

1075 that includes that component, the resulting configuration only includes 

1076 the subset. For example, your config might contain ``dimensions`` if it's 

1077 part of a global config and that subset will be stored. If ``dimensions`` 

1078 can not be found it is assumed that the entire contents of the 

1079 configuration should be used. 

1080 

1081 Default values are read from the environment or supplied search paths 

1082 using the default configuration file name specified in the subclass. 

1083 This allows a configuration class to be instantiated without any 

1084 additional arguments. 

1085 

1086 Additional validation can be specified to check for keys that are mandatory 

1087 in the configuration. 

1088 

1089 Parameters 

1090 ---------- 

1091 other : `Config` or `str` or `dict` 

1092 Argument specifying the configuration information as understood 

1093 by `Config` 

1094 validate : `bool`, optional 

1095 If `True` required keys will be checked to ensure configuration 

1096 consistency. 

1097 mergeDefaults : `bool`, optional 

1098 If `True` defaults will be read and the supplied config will 

1099 be combined with the defaults, with the supplied values taking 

1100 precedence. 

1101 searchPaths : `list` or `tuple`, optional 

1102 Explicit additional paths to search for defaults. They should 

1103 be supplied in priority order. These paths have higher priority 

1104 than those read from the environment in 

1105 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1106 the local file system or URIs, `lsst.resources.ResourcePath`. 

1107 """ 

1108 

1109 component: ClassVar[Optional[str]] = None 

1110 """Component to use from supplied config. Can be None. If specified the 

1111 key is not required. Can be a full dot-separated path to a component. 

1112 """ 

1113 

1114 requiredKeys: ClassVar[Sequence[str]] = () 

1115 """Keys that are required to be specified in the configuration. 

1116 """ 

1117 

1118 defaultConfigFile: ClassVar[Optional[str]] = None 

1119 """Name of the file containing defaults for this config class. 

1120 """ 

1121 

1122 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1123 

1124 # Create a blank object to receive the defaults 

1125 # Once we have the defaults we then update with the external values 

1126 super().__init__() 

1127 

1128 # Create a standard Config rather than subset 

1129 externalConfig = Config(other) 

1130 

1131 # Select the part we need from it 

1132 # To simplify the use of !include we also check for the existence of 

1133 # component.component (since the included files can themselves 

1134 # include the component name) 

1135 if self.component is not None: 1135 ↛ 1144line 1135 didn't jump to line 1144, because the condition on line 1135 was never false

1136 doubled = (self.component, self.component) 

1137 # Must check for double depth first 

1138 if doubled in externalConfig: 1138 ↛ 1139line 1138 didn't jump to line 1139, because the condition on line 1138 was never true

1139 externalConfig = externalConfig[doubled] 

1140 elif self.component in externalConfig: 

1141 externalConfig._data = externalConfig._data[self.component] 

1142 

1143 # Default files read to create this configuration 

1144 self.filesRead = [] 

1145 

1146 # Assume we are not looking up child configurations 

1147 containerKey = None 

1148 

1149 # Sometimes we do not want to merge with defaults. 

1150 if mergeDefaults: 

1151 

1152 # Supplied search paths have highest priority 

1153 fullSearchPath = [] 

1154 if searchPaths: 1154 ↛ 1155line 1154 didn't jump to line 1155, because the condition on line 1154 was never true

1155 fullSearchPath.extend(searchPaths) 

1156 

1157 # Read default paths from environment 

1158 fullSearchPath.extend(self.defaultSearchPaths()) 

1159 

1160 # There are two places to find defaults for this particular config 

1161 # - The "defaultConfigFile" defined in the subclass 

1162 # - The class specified in the "cls" element in the config. 

1163 # Read cls after merging in case it changes. 

1164 if self.defaultConfigFile is not None: 1164 ↛ 1169line 1164 didn't jump to line 1169, because the condition on line 1164 was never false

1165 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1166 

1167 # Can have a class specification in the external config (priority) 

1168 # or from the defaults. 

1169 pytype = None 

1170 if "cls" in externalConfig: 1170 ↛ 1171line 1170 didn't jump to line 1171, because the condition on line 1170 was never true

1171 pytype = externalConfig["cls"] 

1172 elif "cls" in self: 1172 ↛ 1173line 1172 didn't jump to line 1173, because the condition on line 1172 was never true

1173 pytype = self["cls"] 

1174 

1175 if pytype is not None: 1175 ↛ 1176line 1175 didn't jump to line 1176, because the condition on line 1175 was never true

1176 try: 

1177 cls = doImport(pytype) 

1178 except ImportError as e: 

1179 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1180 defaultsFile = cls.defaultConfigFile 

1181 if defaultsFile is not None: 

1182 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1183 

1184 # Get the container key in case we need it 

1185 try: 

1186 containerKey = cls.containerKey 

1187 except AttributeError: 

1188 pass 

1189 

1190 # Now update this object with the external values so that the external 

1191 # values always override the defaults 

1192 self.update(externalConfig) 

1193 if not self.configFile: 1193 ↛ 1199line 1193 didn't jump to line 1199, because the condition on line 1193 was never false

1194 self.configFile = externalConfig.configFile 

1195 

1196 # If this configuration has child configurations of the same 

1197 # config class, we need to expand those defaults as well. 

1198 

1199 if mergeDefaults and containerKey is not None and containerKey in self: 1199 ↛ 1200line 1199 didn't jump to line 1200, because the condition on line 1199 was never true

1200 for idx, subConfig in enumerate(self[containerKey]): 

1201 self[containerKey, idx] = type(self)( 

1202 other=subConfig, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths 

1203 ) 

1204 

1205 if validate: 

1206 self.validate() 

1207 

1208 @classmethod 

1209 def defaultSearchPaths(cls): 

1210 """Read environment to determine search paths to use. 

1211 

1212 Global defaults, at lowest priority, are found in the ``config`` 

1213 directory of the butler source tree. Additional defaults can be 

1214 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1215 which is a PATH-like variable where paths at the front of the list 

1216 have priority over those later. 

1217 

1218 Returns 

1219 ------- 

1220 paths : `list` 

1221 Returns a list of paths to search. The returned order is in 

1222 priority with the highest priority paths first. The butler config 

1223 configuration resources will not be included here but will 

1224 always be searched last. 

1225 

1226 Notes 

1227 ----- 

1228 The environment variable is split on the standard ``:`` path separator. 

1229 This currently makes it incompatible with usage of URIs. 

1230 """ 

1231 # We can pick up defaults from multiple search paths 

1232 # We fill defaults by using the butler config path and then 

1233 # the config path environment variable in reverse order. 

1234 defaultsPaths: List[Union[str, ResourcePath]] = [] 

1235 

1236 if CONFIG_PATH in os.environ: 1236 ↛ 1237line 1236 didn't jump to line 1237, because the condition on line 1236 was never true

1237 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1238 defaultsPaths.extend(externalPaths) 

1239 

1240 # Add the package defaults as a resource 

1241 defaultsPaths.append(ResourcePath(f"resource://{cls.resourcesPackage}/configs", forceDirectory=True)) 

1242 return defaultsPaths 

1243 

1244 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1245 """Search the supplied paths, merging the configuration values. 

1246 

1247 The values read will override values currently stored in the object. 

1248 Every file found in the path will be read, such that the earlier 

1249 path entries have higher priority. 

1250 

1251 Parameters 

1252 ---------- 

1253 searchPaths : `list` of `lsst.resources.ResourcePath`, `str` 

1254 Paths to search for the supplied configFile. This path 

1255 is the priority order, such that files read from the 

1256 first path entry will be selected over those read from 

1257 a later path. Can contain `str` referring to the local file 

1258 system or a URI string. 

1259 configFile : `lsst.resources.ResourcePath` 

1260 File to locate in path. If absolute path it will be read 

1261 directly and the search path will not be used. Can be a URI 

1262 to an explicit resource (which will ignore the search path) 

1263 which is assumed to exist. 

1264 """ 

1265 uri = ResourcePath(configFile) 

1266 if uri.isabs() and uri.exists(): 1266 ↛ 1268line 1266 didn't jump to line 1268, because the condition on line 1266 was never true

1267 # Assume this resource exists 

1268 self._updateWithOtherConfigFile(configFile) 

1269 self.filesRead.append(configFile) 

1270 else: 

1271 # Reverse order so that high priority entries 

1272 # update the object last. 

1273 for pathDir in reversed(searchPaths): 

1274 if isinstance(pathDir, (str, ResourcePath)): 1274 ↛ 1281line 1274 didn't jump to line 1281, because the condition on line 1274 was never false

1275 pathDir = ResourcePath(pathDir, forceDirectory=True) 

1276 file = pathDir.join(configFile) 

1277 if file.exists(): 1277 ↛ 1273line 1277 didn't jump to line 1273, because the condition on line 1277 was never false

1278 self.filesRead.append(file) 

1279 self._updateWithOtherConfigFile(file) 

1280 else: 

1281 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1282 

1283 def _updateWithOtherConfigFile(self, file): 

1284 """Read in some defaults and update. 

1285 

1286 Update the configuration by reading the supplied file as a config 

1287 of this class, and merging such that these values override the 

1288 current values. Contents of the external config are not validated. 

1289 

1290 Parameters 

1291 ---------- 

1292 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

1293 Entity that can be converted to a `ConfigSubset`. 

1294 """ 

1295 # Use this class to read the defaults so that subsetting can happen 

1296 # correctly. 

1297 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1298 self.update(externalConfig) 

1299 

1300 def validate(self): 

1301 """Check that mandatory keys are present in this configuration. 

1302 

1303 Ignored if ``requiredKeys`` is empty. 

1304 """ 

1305 # Validation 

1306 missing = [k for k in self.requiredKeys if k not in self._data] 

1307 if missing: 1307 ↛ 1308line 1307 didn't jump to line 1308, because the condition on line 1307 was never true

1308 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")