Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from pathlib import Path 

37from yaml.representer import Representer 

38import io 

39from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union 

40 

41from lsst.utils import doImport 

42from ._butlerUri import ButlerURI 

43 

44yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

45 

46 

47# Config module logger 

48log = logging.getLogger(__name__) 

49 

50# PATH-like environment variable to use for defaults. 

51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

52 

53try: 

54 yamlLoader = yaml.CSafeLoader 

55except AttributeError: 

56 # Not all installations have the C library 

57 # (but assume for mypy's sake that they're the same) 

58 yamlLoader = yaml.SafeLoader # type: ignore 

59 

60 

61class Loader(yamlLoader): 

62 """YAML Loader that supports file include directives. 

63 

64 Uses ``!include`` directive in a YAML file to point to another 

65 YAML file to be included. The path in the include directive is relative 

66 to the file containing that directive. 

67 

68 storageClasses: !include storageClasses.yaml 

69 

70 Examples 

71 -------- 

72 >>> with open("document.yaml", "r") as f: 

73 data = yaml.load(f, Loader=Loader) 

74 

75 Notes 

76 ----- 

77 See https://davidchall.github.io/yaml-includes.html 

78 """ 

79 

80 def __init__(self, stream): 

81 super().__init__(stream) 

82 # if this is a string and not a stream we may well lack a name 

83 try: 

84 self._root = ButlerURI(stream.name) 

85 except AttributeError: 

86 # No choice but to assume a local filesystem 

87 self._root = ButlerURI("no-file.yaml") 

88 Loader.add_constructor("!include", Loader.include) 

89 

90 def include(self, node): 

91 result: Union[List[Any], Dict[str, Any]] 

92 if isinstance(node, yaml.ScalarNode): 

93 return self.extractFile(self.construct_scalar(node)) 

94 

95 elif isinstance(node, yaml.SequenceNode): 

96 result = [] 

97 for filename in self.construct_sequence(node): 

98 result.append(self.extractFile(filename)) 

99 return result 

100 

101 elif isinstance(node, yaml.MappingNode): 

102 result = {} 

103 for k, v in self.construct_mapping(node).items(): 

104 result[k] = self.extractFile(v) 

105 return result 

106 

107 else: 

108 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

109 raise yaml.constructor.ConstructorError 

110 

111 def extractFile(self, filename): 

112 # It is possible for the !include to point to an explicit URI 

113 # instead of a relative URI, therefore we first see if it is 

114 # scheme-less or not. If it has a scheme we use it directly 

115 # if it is scheme-less we use it relative to the file root. 

116 requesteduri = ButlerURI(filename, forceAbsolute=False) 

117 

118 if requesteduri.scheme: 

119 fileuri = requesteduri 

120 else: 

121 fileuri = self._root.updatedFile(filename) 

122 

123 log.debug("Opening YAML file via !include: %s", fileuri) 

124 

125 # Read all the data from the resource 

126 data = fileuri.read() 

127 

128 # Store the bytes into a BytesIO so we can attach a .name 

129 stream = io.BytesIO(data) 

130 stream.name = fileuri.geturl() 

131 return yaml.load(stream, Loader) 

132 

133 

134class Config(collections.abc.MutableMapping): 

135 r"""Implements a datatype that is used by `Butler` for configuration. 

136 

137 It is essentially a `dict` with key/value pairs, including nested dicts 

138 (as values). In fact, it can be initialized with a `dict`. 

139 This is explained next: 

140 

141 Config extends the `dict` api so that hierarchical values may be accessed 

142 with delimited notation or as a tuple. If a string is given the delimiter 

143 is picked up from the first character in that string. For example, 

144 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

145 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

146 If the first character is alphanumeric, no delimiter will be used. 

147 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

148 Unicode characters can be used as the delimiter for distinctiveness if 

149 required. 

150 

151 If a key in the hierarchy starts with a non-alphanumeric character care 

152 should be used to ensure that either the tuple interface is used or 

153 a distinct delimiter is always given in string form. 

154 

155 Finally, the delimiter can be escaped if it is part of a key and also 

156 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

157 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

158 always better to use a different delimiter in these cases. 

159 

160 Note that adding a multi-level key implicitly creates any nesting levels 

161 that do not exist, but removing multi-level keys does not automatically 

162 remove empty nesting levels. As a result: 

163 

164 >>> c = Config() 

165 >>> c[".a.b"] = 1 

166 >>> del c[".a.b"] 

167 >>> c["a"] 

168 Config({'a': {}}) 

169 

170 Storage formats supported: 

171 

172 - yaml: read and write is supported. 

173 - json: read and write is supported but no ``!include`` directive. 

174 

175 Parameters 

176 ---------- 

177 other : `str` or `Config` or `dict` or `ButlerURI` or `pathlib.Path` 

178 Other source of configuration, can be: 

179 

180 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

181 with ".yaml". 

182 - (`Config`) Copies the other Config's values into this one. 

183 - (`dict`) Copies the values from the dict into this Config. 

184 

185 If `None` is provided an empty `Config` will be created. 

186 """ 

187 

188 _D: str = "→" 

189 """Default internal delimiter to use for components in the hierarchy when 

190 constructing keys for external use (see `Config.names()`).""" 

191 

192 includeKey: ClassVar[str] = "includeConfigs" 

193 """Key used to indicate that another config should be included at this 

194 part of the hierarchy.""" 

195 

196 resourcesPackage: str = "lsst.daf.butler" 

197 """Package to search for default configuration data. The resources 

198 themselves will be within a ``configs`` resource hierarchy.""" 

199 

200 def __init__(self, other=None): 

201 self._data: Dict[str, Any] = {} 

202 self.configFile = None 

203 

204 if other is None: 

205 return 

206 

207 if isinstance(other, Config): 

208 self._data = copy.deepcopy(other._data) 

209 self.configFile = other.configFile 

210 elif isinstance(other, collections.abc.Mapping): 

211 self.update(other) 

212 elif isinstance(other, (str, ButlerURI, Path)): 212 ↛ 219line 212 didn't jump to line 219, because the condition on line 212 was never false

213 # if other is a string, assume it is a file path/URI 

214 self.__initFromUri(other) 

215 self._processExplicitIncludes() 

216 else: 

217 # if the config specified by other could not be recognized raise 

218 # a runtime error. 

219 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

220 

221 def ppprint(self): 

222 """Return config as formatted readable string. 

223 

224 Examples 

225 -------- 

226 use: ``pdb> print(myConfigObject.ppprint())`` 

227 

228 Returns 

229 ------- 

230 s : `str` 

231 A prettyprint formatted string representing the config 

232 """ 

233 return pprint.pformat(self._data, indent=2, width=1) 

234 

235 def __repr__(self): 

236 return f"{type(self).__name__}({self._data!r})" 

237 

238 def __str__(self): 

239 return self.ppprint() 

240 

241 def __len__(self): 

242 return len(self._data) 

243 

244 def __iter__(self): 

245 return iter(self._data) 

246 

247 def copy(self): 

248 return type(self)(self) 

249 

250 @classmethod 

251 def fromString(cls, string: str, format: str = "yaml") -> Config: 

252 """Create a new Config instance from a serialized string. 

253 

254 Parameters 

255 ---------- 

256 string : `str` 

257 String containing content in specified format 

258 format : `str`, optional 

259 Format of the supplied string. Can be ``json`` or ``yaml``. 

260 

261 Returns 

262 ------- 

263 c : `Config` 

264 Newly-constructed Config. 

265 """ 

266 if format == "yaml": 

267 new_config = cls().__initFromYaml(string) 

268 elif format == "json": 

269 new_config = cls().__initFromJson(string) 

270 else: 

271 raise ValueError(f"Unexpected format of string: {format}") 

272 new_config._processExplicitIncludes() 

273 return new_config 

274 

275 @classmethod 

276 def fromYaml(cls, string: str) -> Config: 

277 """Create a new Config instance from a YAML string. 

278 

279 Parameters 

280 ---------- 

281 string : `str` 

282 String containing content in YAML format 

283 

284 Returns 

285 ------- 

286 c : `Config` 

287 Newly-constructed Config. 

288 """ 

289 return cls.fromString(string, format="yaml") 

290 

291 def __initFromUri(self, path: Union[str, ButlerURI, Path]) -> None: 

292 """Load a file from a path or an URI. 

293 

294 Parameters 

295 ---------- 

296 path : `str` 

297 Path or a URI to a persisted config file. 

298 """ 

299 uri = ButlerURI(path) 

300 ext = uri.getExtension() 

301 if ext == ".yaml": 301 ↛ 308line 301 didn't jump to line 308, because the condition on line 301 was never false

302 log.debug("Opening YAML config file: %s", uri.geturl()) 

303 content = uri.read() 

304 # Use a stream so we can name it 

305 stream = io.BytesIO(content) 

306 stream.name = uri.geturl() 

307 self.__initFromYaml(stream) 

308 elif ext == ".json": 

309 log.debug("Opening JSON config file: %s", uri.geturl()) 

310 content = uri.read() 

311 self.__initFromJson(content) 

312 else: 

313 # This URI does not have a valid extension. It might be because 

314 # we ended up with a directory and not a file. Before we complain 

315 # about an extension, do an existence check. No need to do 

316 # the (possibly expensive) existence check in the default code 

317 # path above because we will find out soon enough that the file 

318 # is not there. 

319 if not uri.exists(): 

320 raise FileNotFoundError(f"Config location {uri} does not exist.") 

321 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

322 self.configFile = uri 

323 

324 def __initFromYaml(self, stream): 

325 """Load a YAML config from any readable stream that contains one. 

326 

327 Parameters 

328 ---------- 

329 stream: `IO` or `str` 

330 Stream to pass to the YAML loader. Accepts anything that 

331 `yaml.load` accepts. This can include a string as well as an 

332 IO stream. 

333 

334 Raises 

335 ------ 

336 yaml.YAMLError 

337 If there is an error loading the file. 

338 """ 

339 content = yaml.load(stream, Loader=Loader) 

340 if content is None: 340 ↛ 341line 340 didn't jump to line 341, because the condition on line 340 was never true

341 content = {} 

342 self._data = content 

343 return self 

344 

345 def __initFromJson(self, stream): 

346 """Load a JSON config from any readable stream that contains one. 

347 

348 Parameters 

349 ---------- 

350 stream: `IO` or `str` 

351 Stream to pass to the JSON loader. This can include a string as 

352 well as an IO stream. 

353 

354 Raises 

355 ------ 

356 TypeError: 

357 Raised if there is an error loading the content. 

358 """ 

359 if isinstance(stream, (bytes, str)): 

360 content = json.loads(stream) 

361 else: 

362 content = json.load(stream) 

363 if content is None: 

364 content = {} 

365 self._data = content 

366 return self 

367 

368 def _processExplicitIncludes(self): 

369 """Scan through the configuration searching for the special includes. 

370 

371 Looks for ``includeConfigs`` directive and processes the includes. 

372 """ 

373 # Search paths for config files 

374 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

375 if self.configFile is not None: 375 ↛ 383line 375 didn't jump to line 383, because the condition on line 375 was never false

376 if isinstance(self.configFile, ButlerURI): 376 ↛ 379line 376 didn't jump to line 379, because the condition on line 376 was never false

377 configDir = self.configFile.dirname() 

378 else: 

379 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

380 searchPaths.append(configDir) 

381 

382 # Ensure we know what delimiter to use 

383 names = self.nameTuples() 

384 for path in names: 

385 if path[-1] == self.includeKey: 385 ↛ 387line 385 didn't jump to line 387, because the condition on line 385 was never true

386 

387 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

388 basePath = path[:-1] 

389 

390 # Extract the includes and then delete them from the config 

391 includes = self[path] 

392 del self[path] 

393 

394 # Be consistent and convert to a list 

395 if not isinstance(includes, list): 

396 includes = [includes] 

397 

398 # Read each file assuming it is a reference to a file 

399 # The file can be relative to config file or cwd 

400 # ConfigSubset search paths are not used 

401 subConfigs = [] 

402 for fileName in includes: 

403 # Expand any shell variables -- this could be URI 

404 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

405 found = None 

406 if fileName.isabs(): 

407 found = fileName 

408 else: 

409 for dir in searchPaths: 

410 if isinstance(dir, ButlerURI): 

411 specific = dir.join(fileName.path) 

412 # Remote resource check might be expensive 

413 if specific.exists(): 

414 found = specific 

415 else: 

416 log.warning("Do not understand search path entry '%s' of type %s", 

417 dir, type(dir).__name__) 

418 if not found: 

419 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

420 

421 # Read the referenced Config as a Config 

422 subConfigs.append(type(self)(found)) 

423 

424 # Now we need to merge these sub configs with the current 

425 # information that was present in this node in the config 

426 # tree with precedence given to the explicit values 

427 newConfig = subConfigs.pop(0) 

428 for sc in subConfigs: 

429 newConfig.update(sc) 

430 

431 # Explicit values take precedence 

432 if not basePath: 

433 # This is an include at the root config 

434 newConfig.update(self) 

435 # Replace the current config 

436 self._data = newConfig._data 

437 else: 

438 newConfig.update(self[basePath]) 

439 # And reattach to the base config 

440 self[basePath] = newConfig 

441 

442 @staticmethod 

443 def _splitIntoKeys(key): 

444 r"""Split the argument for get/set/in into a hierarchical list. 

445 

446 Parameters 

447 ---------- 

448 key : `str` or iterable 

449 Argument given to get/set/in. If an iterable is provided it will 

450 be converted to a list. If the first character of the string 

451 is not an alphanumeric character then it will be used as the 

452 delimiter for the purposes of splitting the remainder of the 

453 string. If the delimiter is also in one of the keys then it 

454 can be escaped using ``\``. There is no default delimiter. 

455 

456 Returns 

457 ------- 

458 keys : `list` 

459 Hierarchical keys as a `list`. 

460 """ 

461 if isinstance(key, str): 

462 if not key[0].isalnum(): 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true

463 d = key[0] 

464 key = key[1:] 

465 else: 

466 return [key, ] 

467 escaped = f"\\{d}" 

468 temp = None 

469 if escaped in key: 

470 # Complain at the attempt to escape the escape 

471 doubled = fr"\{escaped}" 

472 if doubled in key: 

473 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

474 " is not yet supported.") 

475 # Replace with a character that won't be in the string 

476 temp = "\r" 

477 if temp in key or d == temp: 

478 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

479 " delimiter if escaping the delimiter") 

480 key = key.replace(escaped, temp) 

481 hierarchy = key.split(d) 

482 if temp: 

483 hierarchy = [h.replace(temp, d) for h in hierarchy] 

484 return hierarchy 

485 elif isinstance(key, collections.abc.Iterable): 485 ↛ 489line 485 didn't jump to line 489, because the condition on line 485 was never false

486 return list(key) 

487 else: 

488 # Not sure what this is so try it anyway 

489 return [key, ] 

490 

491 def _getKeyHierarchy(self, name): 

492 """Retrieve the key hierarchy for accessing the Config. 

493 

494 Parameters 

495 ---------- 

496 name : `str` or `tuple` 

497 Delimited string or `tuple` of hierarchical keys. 

498 

499 Returns 

500 ------- 

501 hierarchy : `list` of `str` 

502 Hierarchy to use as a `list`. If the name is available directly 

503 as a key in the Config it will be used regardless of the presence 

504 of any nominal delimiter. 

505 """ 

506 if name in self._data: 

507 keys = [name, ] 

508 else: 

509 keys = self._splitIntoKeys(name) 

510 return keys 

511 

512 def _findInHierarchy(self, keys, create=False): 

513 """Look for hierarchy of keys in Config. 

514 

515 Parameters 

516 ---------- 

517 keys : `list` or `tuple` 

518 Keys to search in hierarchy. 

519 create : `bool`, optional 

520 If `True`, if a part of the hierarchy does not exist, insert an 

521 empty `dict` into the hierarchy. 

522 

523 Returns 

524 ------- 

525 hierarchy : `list` 

526 List of the value corresponding to each key in the supplied 

527 hierarchy. Only keys that exist in the hierarchy will have 

528 a value. 

529 complete : `bool` 

530 `True` if the full hierarchy exists and the final element 

531 in ``hierarchy`` is the value of relevant value. 

532 """ 

533 d = self._data 

534 

535 def checkNextItem(k, d, create): 

536 """See if k is in d and if it is return the new child.""" 

537 nextVal = None 

538 isThere = False 

539 if d is None: 539 ↛ 541line 539 didn't jump to line 541, because the condition on line 539 was never true

540 # We have gone past the end of the hierarchy 

541 pass 

542 elif isinstance(d, collections.abc.Sequence): 542 ↛ 547line 542 didn't jump to line 547, because the condition on line 542 was never true

543 # Check sequence first because for lists 

544 # __contains__ checks whether value is found in list 

545 # not whether the index exists in list. When we traverse 

546 # the hierarchy we are interested in the index. 

547 try: 

548 nextVal = d[int(k)] 

549 isThere = True 

550 except IndexError: 

551 pass 

552 except ValueError: 

553 isThere = k in d 

554 elif k in d: 

555 nextVal = d[k] 

556 isThere = True 

557 elif create: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true

558 d[k] = {} 

559 nextVal = d[k] 

560 isThere = True 

561 return nextVal, isThere 

562 

563 hierarchy = [] 

564 complete = True 

565 for k in keys: 

566 d, isThere = checkNextItem(k, d, create) 

567 if isThere: 

568 hierarchy.append(d) 

569 else: 

570 complete = False 

571 break 

572 

573 return hierarchy, complete 

574 

575 def __getitem__(self, name): 

576 # Override the split for the simple case where there is an exact 

577 # match. This allows `Config.items()` to work via a simple 

578 # __iter__ implementation that returns top level keys of 

579 # self._data. 

580 keys = self._getKeyHierarchy(name) 

581 

582 hierarchy, complete = self._findInHierarchy(keys) 

583 if not complete: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true

584 raise KeyError(f"{name} not found") 

585 data = hierarchy[-1] 

586 

587 if isinstance(data, collections.abc.Mapping): 

588 data = Config(data) 

589 # Ensure that child configs inherit the parent internal delimiter 

590 if self._D != Config._D: 590 ↛ 591line 590 didn't jump to line 591, because the condition on line 590 was never true

591 data._D = self._D 

592 return data 

593 

594 def __setitem__(self, name, value): 

595 keys = self._getKeyHierarchy(name) 

596 last = keys.pop() 

597 if isinstance(value, Config): 

598 value = copy.deepcopy(value._data) 

599 

600 hierarchy, complete = self._findInHierarchy(keys, create=True) 

601 if hierarchy: 

602 data = hierarchy[-1] 

603 else: 

604 data = self._data 

605 

606 try: 

607 data[last] = value 

608 except TypeError: 

609 data[int(last)] = value 

610 

611 def __contains__(self, key): 

612 keys = self._getKeyHierarchy(key) 

613 hierarchy, complete = self._findInHierarchy(keys) 

614 return complete 

615 

616 def __delitem__(self, key): 

617 keys = self._getKeyHierarchy(key) 

618 last = keys.pop() 

619 hierarchy, complete = self._findInHierarchy(keys) 

620 if complete: 620 ↛ 627line 620 didn't jump to line 627, because the condition on line 620 was never false

621 if hierarchy: 621 ↛ 622line 621 didn't jump to line 622, because the condition on line 621 was never true

622 data = hierarchy[-1] 

623 else: 

624 data = self._data 

625 del data[last] 

626 else: 

627 raise KeyError(f"{key} not found in Config") 

628 

629 def update(self, other): 

630 """Update config from other `Config` or `dict`. 

631 

632 Like `dict.update()`, but will add or modify keys in nested dicts, 

633 instead of overwriting the nested dict entirely. 

634 

635 Parameters 

636 ---------- 

637 other : `dict` or `Config` 

638 Source of configuration: 

639 

640 Examples 

641 -------- 

642 >>> c = Config({"a": {"b": 1}}) 

643 >>> c.update({"a": {"c": 2}}) 

644 >>> print(c) 

645 {'a': {'b': 1, 'c': 2}} 

646 

647 >>> foo = {"a": {"b": 1}} 

648 >>> foo.update({"a": {"c": 2}}) 

649 >>> print(foo) 

650 {'a': {'c': 2}} 

651 """ 

652 def doUpdate(d, u): 

653 if not isinstance(u, collections.abc.Mapping) or \ 653 ↛ 655line 653 didn't jump to line 655, because the condition on line 653 was never true

654 not isinstance(d, collections.abc.MutableMapping): 

655 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

656 for k, v in u.items(): 

657 if isinstance(v, collections.abc.Mapping): 

658 d[k] = doUpdate(d.get(k, {}), v) 

659 else: 

660 d[k] = v 

661 return d 

662 doUpdate(self._data, other) 

663 

664 def merge(self, other): 

665 """Merge another Config into this one. 

666 

667 Like `Config.update()`, but will add keys & values from other that 

668 DO NOT EXIST in self. 

669 

670 Keys and values that already exist in self will NOT be overwritten. 

671 

672 Parameters 

673 ---------- 

674 other : `dict` or `Config` 

675 Source of configuration: 

676 """ 

677 if not isinstance(other, collections.abc.Mapping): 

678 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}") 

679 

680 # Convert the supplied mapping to a Config for consistency 

681 # This will do a deepcopy if it is already a Config 

682 otherCopy = Config(other) 

683 otherCopy.update(self) 

684 self._data = otherCopy._data 

685 

686 def nameTuples(self, topLevelOnly=False): 

687 """Get tuples representing the name hierarchies of all keys. 

688 

689 The tuples returned from this method are guaranteed to be usable 

690 to access items in the configuration object. 

691 

692 Parameters 

693 ---------- 

694 topLevelOnly : `bool`, optional 

695 If False, the default, a full hierarchy of names is returned. 

696 If True, only the top level are returned. 

697 

698 Returns 

699 ------- 

700 names : `list` of `tuple` of `str` 

701 List of all names present in the `Config` where each element 

702 in the list is a `tuple` of strings representing the hierarchy. 

703 """ 

704 if topLevelOnly: 704 ↛ 705line 704 didn't jump to line 705, because the condition on line 704 was never true

705 return list((k,) for k in self) 

706 

707 def getKeysAsTuples(d, keys, base): 

708 if isinstance(d, collections.abc.Sequence): 

709 theseKeys = range(len(d)) 

710 else: 

711 theseKeys = d.keys() 

712 for key in theseKeys: 

713 val = d[key] 

714 levelKey = base + (key,) if base is not None else (key,) 

715 keys.append(levelKey) 

716 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

717 and not isinstance(val, str): 

718 getKeysAsTuples(val, keys, levelKey) 

719 keys: List[Tuple[str, ...]] = [] 

720 getKeysAsTuples(self._data, keys, None) 

721 return keys 

722 

723 def names(self, topLevelOnly=False, delimiter=None): 

724 """Get a delimited name of all the keys in the hierarchy. 

725 

726 The values returned from this method are guaranteed to be usable 

727 to access items in the configuration object. 

728 

729 Parameters 

730 ---------- 

731 topLevelOnly : `bool`, optional 

732 If False, the default, a full hierarchy of names is returned. 

733 If True, only the top level are returned. 

734 delimiter : `str`, optional 

735 Delimiter to use when forming the keys. If the delimiter is 

736 present in any of the keys, it will be escaped in the returned 

737 names. If `None` given a delimiter will be automatically provided. 

738 The delimiter can not be alphanumeric. 

739 

740 Returns 

741 ------- 

742 names : `list` of `str` 

743 List of all names present in the `Config`. 

744 

745 Notes 

746 ----- 

747 This is different than the built-in method `dict.keys`, which will 

748 return only the first level keys. 

749 

750 Raises 

751 ------ 

752 ValueError: 

753 The supplied delimiter is alphanumeric. 

754 """ 

755 if topLevelOnly: 

756 return list(self.keys()) 

757 

758 # Get all the tuples of hierarchical keys 

759 nameTuples = self.nameTuples() 

760 

761 if delimiter is not None and delimiter.isalnum(): 

762 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

763 

764 if delimiter is None: 

765 # Start with something, and ensure it does not need to be 

766 # escaped (it is much easier to understand if not escaped) 

767 delimiter = self._D 

768 

769 # Form big string for easy check of delimiter clash 

770 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

771 

772 # Try a delimiter and keep trying until we get something that 

773 # works. 

774 ntries = 0 

775 while delimiter in combined: 

776 log.debug("Delimiter '%s' could not be used. Trying another.", delimiter) 

777 ntries += 1 

778 

779 if ntries > 100: 

780 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

781 

782 # try another one 

783 while True: 

784 delimiter = chr(ord(delimiter)+1) 

785 if not delimiter.isalnum(): 

786 break 

787 

788 log.debug("Using delimiter %r", delimiter) 

789 

790 # Form the keys, escaping the delimiter if necessary 

791 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

792 for k in nameTuples] 

793 return strings 

794 

795 def asArray(self, name): 

796 """Get a value as an array. 

797 

798 May contain one or more elements. 

799 

800 Parameters 

801 ---------- 

802 name : `str` 

803 Key to use to retrieve value. 

804 

805 Returns 

806 ------- 

807 array : `collections.abc.Sequence` 

808 The value corresponding to name, but guaranteed to be returned 

809 as a list with at least one element. If the value is a 

810 `~collections.abc.Sequence` (and not a `str`) the value itself 

811 will be returned, else the value will be the first element. 

812 """ 

813 val = self.get(name) 

814 if isinstance(val, str): 

815 val = [val] 

816 elif not isinstance(val, collections.abc.Sequence): 

817 val = [val] 

818 return val 

819 

820 def __eq__(self, other): 

821 if isinstance(other, Config): 

822 other = other._data 

823 return self._data == other 

824 

825 def __ne__(self, other): 

826 if isinstance(other, Config): 

827 other = other._data 

828 return self._data != other 

829 

830 ####### 

831 # i/o # 

832 

833 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

834 """Write the config to an output stream. 

835 

836 Parameters 

837 ---------- 

838 output : `IO`, optional 

839 The stream to use for output. If `None` the serialized content 

840 will be returned. 

841 format : `str`, optional 

842 The format to use for the output. Can be "yaml" or "json". 

843 

844 Returns 

845 ------- 

846 serialized : `str` or `None` 

847 If a stream was given the stream will be used and the return 

848 value will be `None`. If the stream was `None` the 

849 serialization will be returned as a string. 

850 """ 

851 if format == "yaml": 

852 return yaml.safe_dump(self._data, output, default_flow_style=False) 

853 elif format == "json": 

854 if output is not None: 

855 json.dump(self._data, output, ensure_ascii=False) 

856 return None 

857 else: 

858 return json.dumps(self._data, ensure_ascii=False) 

859 raise ValueError(f"Unsupported format for Config serialization: {format}") 

860 

861 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

862 defaultFileName: str = "butler.yaml", 

863 overwrite: bool = True) -> None: 

864 """Write the config to location pointed to by given URI. 

865 

866 Currently supports 's3' and 'file' URI schemes. 

867 

868 Parameters 

869 ---------- 

870 uri: `str` or `ButlerURI` 

871 URI of location where the Config will be written. 

872 updateFile : bool, optional 

873 If True and uri does not end on a filename with extension, will 

874 append `defaultFileName` to the target uri. True by default. 

875 defaultFileName : bool, optional 

876 The file name that will be appended to target uri if updateFile is 

877 True and uri does not end on a file with an extension. 

878 overwrite : bool, optional 

879 If True the configuration will be written even if it already 

880 exists at that location. 

881 """ 

882 # Make local copy of URI or create new one 

883 uri = ButlerURI(uri) 

884 

885 if updateFile and not uri.getExtension(): 

886 uri = uri.updatedFile(defaultFileName) 

887 

888 # Try to work out the format from the extension 

889 ext = uri.getExtension() 

890 format = ext[1:].lower() 

891 

892 output = self.dump(format=format) 

893 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

894 uri.write(output.encode(), overwrite=overwrite) 

895 self.configFile = uri 

896 

897 @staticmethod 

898 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True, toMerge=None): 

899 """Update specific config parameters. 

900 

901 Allows for named parameters to be set to new values in bulk, and 

902 for other values to be set by copying from a reference config. 

903 

904 Assumes that the supplied config is compatible with ``configType`` 

905 and will attach the updated values to the supplied config by 

906 looking for the related component key. It is assumed that 

907 ``config`` and ``full`` are from the same part of the 

908 configuration hierarchy. 

909 

910 Parameters 

911 ---------- 

912 configType : `ConfigSubset` 

913 Config type to use to extract relevant items from ``config``. 

914 config : `Config` 

915 A `Config` to update. Only the subset understood by 

916 the supplied `ConfigSubset` will be modified. Default values 

917 will not be inserted and the content will not be validated 

918 since mandatory keys are allowed to be missing until 

919 populated later by merging. 

920 full : `Config` 

921 A complete config with all defaults expanded that can be 

922 converted to a ``configType``. Read-only and will not be 

923 modified by this method. Values are read from here if 

924 ``toCopy`` is defined. 

925 

926 Repository-specific options that should not be obtained 

927 from defaults when Butler instances are constructed 

928 should be copied from ``full`` to ``config``. 

929 toUpdate : `dict`, optional 

930 A `dict` defining the keys to update and the new value to use. 

931 The keys and values can be any supported by `Config` 

932 assignment. 

933 toCopy : `tuple`, optional 

934 `tuple` of keys whose values should be copied from ``full`` 

935 into ``config``. 

936 overwrite : `bool`, optional 

937 If `False`, do not modify a value in ``config`` if the key 

938 already exists. Default is always to overwrite. 

939 toMerge : `tuple`, optional 

940 Keys to merge content from full to config without overwriting 

941 pre-existing values. Only works if the key refers to a hierarchy. 

942 The ``overwrite`` flag is ignored. 

943 

944 Raises 

945 ------ 

946 ValueError 

947 Neither ``toUpdate``, ``toCopy`` nor ``toMerge`` were defined. 

948 """ 

949 if toUpdate is None and toCopy is None and toMerge is None: 

950 raise ValueError("At least one of toUpdate, toCopy, or toMerge parameters must be set.") 

951 

952 # If this is a parent configuration then we need to ensure that 

953 # the supplied config has the relevant component key in it. 

954 # If this is a parent configuration we add in the stub entry 

955 # so that the ConfigSubset constructor will do the right thing. 

956 # We check full for this since that is guaranteed to be complete. 

957 if configType.component in full and configType.component not in config: 

958 config[configType.component] = {} 

959 

960 # Extract the part of the config we wish to update 

961 localConfig = configType(config, mergeDefaults=False, validate=False) 

962 

963 if toUpdate: 

964 for key, value in toUpdate.items(): 

965 if key in localConfig and not overwrite: 

966 log.debug("Not overriding key '%s' with value '%s' in config %s", 

967 key, value, localConfig.__class__.__name__) 

968 else: 

969 localConfig[key] = value 

970 

971 if toCopy or toMerge: 

972 localFullConfig = configType(full, mergeDefaults=False) 

973 

974 if toCopy: 

975 for key in toCopy: 

976 if key in localConfig and not overwrite: 

977 log.debug("Not overriding key '%s' from defaults in config %s", 

978 key, localConfig.__class__.__name__) 

979 else: 

980 localConfig[key] = localFullConfig[key] 

981 if toMerge: 

982 for key in toMerge: 

983 if key in localConfig: 

984 # Get the node from the config to do the merge 

985 # but then have to reattach to the config. 

986 subset = localConfig[key] 

987 subset.merge(localFullConfig[key]) 

988 localConfig[key] = subset 

989 else: 

990 localConfig[key] = localFullConfig[key] 

991 

992 # Reattach to parent if this is a child config 

993 if configType.component in config: 

994 config[configType.component] = localConfig 

995 else: 

996 config.update(localConfig) 

997 

998 def toDict(self): 

999 """Convert a `Config` to a standalone hierarchical `dict`. 

1000 

1001 Returns 

1002 ------- 

1003 d : `dict` 

1004 The standalone hierarchical `dict` with any `Config` classes 

1005 in the hierarchy converted to `dict`. 

1006 

1007 Notes 

1008 ----- 

1009 This can be useful when passing a Config to some code that 

1010 expects native Python types. 

1011 """ 

1012 output = copy.deepcopy(self._data) 

1013 for k, v in output.items(): 

1014 if isinstance(v, Config): 

1015 v = v.toDict() 

1016 output[k] = v 

1017 return output 

1018 

1019 

1020class ConfigSubset(Config): 

1021 """Config representing a subset of a more general configuration. 

1022 

1023 Subclasses define their own component and when given a configuration 

1024 that includes that component, the resulting configuration only includes 

1025 the subset. For example, your config might contain ``dimensions`` if it's 

1026 part of a global config and that subset will be stored. If ``dimensions`` 

1027 can not be found it is assumed that the entire contents of the 

1028 configuration should be used. 

1029 

1030 Default values are read from the environment or supplied search paths 

1031 using the default configuration file name specified in the subclass. 

1032 This allows a configuration class to be instantiated without any 

1033 additional arguments. 

1034 

1035 Additional validation can be specified to check for keys that are mandatory 

1036 in the configuration. 

1037 

1038 Parameters 

1039 ---------- 

1040 other : `Config` or `str` or `dict` 

1041 Argument specifying the configuration information as understood 

1042 by `Config` 

1043 validate : `bool`, optional 

1044 If `True` required keys will be checked to ensure configuration 

1045 consistency. 

1046 mergeDefaults : `bool`, optional 

1047 If `True` defaults will be read and the supplied config will 

1048 be combined with the defaults, with the supplied valiues taking 

1049 precedence. 

1050 searchPaths : `list` or `tuple`, optional 

1051 Explicit additional paths to search for defaults. They should 

1052 be supplied in priority order. These paths have higher priority 

1053 than those read from the environment in 

1054 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1055 the local file system or URIs, `ButlerURI`. 

1056 """ 

1057 

1058 component: ClassVar[Optional[str]] = None 

1059 """Component to use from supplied config. Can be None. If specified the 

1060 key is not required. Can be a full dot-separated path to a component. 

1061 """ 

1062 

1063 requiredKeys: ClassVar[Sequence[str]] = () 

1064 """Keys that are required to be specified in the configuration. 

1065 """ 

1066 

1067 defaultConfigFile: ClassVar[Optional[str]] = None 

1068 """Name of the file containing defaults for this config class. 

1069 """ 

1070 

1071 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1072 

1073 # Create a blank object to receive the defaults 

1074 # Once we have the defaults we then update with the external values 

1075 super().__init__() 

1076 

1077 # Create a standard Config rather than subset 

1078 externalConfig = Config(other) 

1079 

1080 # Select the part we need from it 

1081 # To simplify the use of !include we also check for the existence of 

1082 # component.component (since the included files can themselves 

1083 # include the component name) 

1084 if self.component is not None: 1084 ↛ 1093line 1084 didn't jump to line 1093, because the condition on line 1084 was never false

1085 doubled = (self.component, self.component) 

1086 # Must check for double depth first 

1087 if doubled in externalConfig: 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true

1088 externalConfig = externalConfig[doubled] 

1089 elif self.component in externalConfig: 

1090 externalConfig._data = externalConfig._data[self.component] 

1091 

1092 # Default files read to create this configuration 

1093 self.filesRead = [] 

1094 

1095 # Assume we are not looking up child configurations 

1096 containerKey = None 

1097 

1098 # Sometimes we do not want to merge with defaults. 

1099 if mergeDefaults: 

1100 

1101 # Supplied search paths have highest priority 

1102 fullSearchPath = [] 

1103 if searchPaths: 1103 ↛ 1104line 1103 didn't jump to line 1104, because the condition on line 1103 was never true

1104 fullSearchPath.extend(searchPaths) 

1105 

1106 # Read default paths from enviroment 

1107 fullSearchPath.extend(self.defaultSearchPaths()) 

1108 

1109 # There are two places to find defaults for this particular config 

1110 # - The "defaultConfigFile" defined in the subclass 

1111 # - The class specified in the "cls" element in the config. 

1112 # Read cls after merging in case it changes. 

1113 if self.defaultConfigFile is not None: 1113 ↛ 1118line 1113 didn't jump to line 1118, because the condition on line 1113 was never false

1114 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1115 

1116 # Can have a class specification in the external config (priority) 

1117 # or from the defaults. 

1118 pytype = None 

1119 if "cls" in externalConfig: 1119 ↛ 1120line 1119 didn't jump to line 1120, because the condition on line 1119 was never true

1120 pytype = externalConfig["cls"] 

1121 elif "cls" in self: 1121 ↛ 1122line 1121 didn't jump to line 1122, because the condition on line 1121 was never true

1122 pytype = self["cls"] 

1123 

1124 if pytype is not None: 1124 ↛ 1125line 1124 didn't jump to line 1125, because the condition on line 1124 was never true

1125 try: 

1126 cls = doImport(pytype) 

1127 except ImportError as e: 

1128 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1129 defaultsFile = cls.defaultConfigFile 

1130 if defaultsFile is not None: 

1131 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1132 

1133 # Get the container key in case we need it 

1134 try: 

1135 containerKey = cls.containerKey 

1136 except AttributeError: 

1137 pass 

1138 

1139 # Now update this object with the external values so that the external 

1140 # values always override the defaults 

1141 self.update(externalConfig) 

1142 

1143 # If this configuration has child configurations of the same 

1144 # config class, we need to expand those defaults as well. 

1145 

1146 if mergeDefaults and containerKey is not None and containerKey in self: 1146 ↛ 1147line 1146 didn't jump to line 1147, because the condition on line 1146 was never true

1147 for idx, subConfig in enumerate(self[containerKey]): 

1148 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1149 mergeDefaults=mergeDefaults, 

1150 searchPaths=searchPaths) 

1151 

1152 if validate: 

1153 self.validate() 

1154 

1155 @classmethod 

1156 def defaultSearchPaths(cls): 

1157 """Read environment to determine search paths to use. 

1158 

1159 Global defaults, at lowest priority, are found in the ``config`` 

1160 directory of the butler source tree. Additional defaults can be 

1161 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1162 which is a PATH-like variable where paths at the front of the list 

1163 have priority over those later. 

1164 

1165 Returns 

1166 ------- 

1167 paths : `list` 

1168 Returns a list of paths to search. The returned order is in 

1169 priority with the highest priority paths first. The butler config 

1170 configuration resources will not be included here but will 

1171 always be searched last. 

1172 

1173 Notes 

1174 ----- 

1175 The environment variable is split on the standard ``:`` path separator. 

1176 This currently makes it incompatible with usage of URIs. 

1177 """ 

1178 # We can pick up defaults from multiple search paths 

1179 # We fill defaults by using the butler config path and then 

1180 # the config path environment variable in reverse order. 

1181 defaultsPaths: List[Union[str, ButlerURI]] = [] 

1182 

1183 if CONFIG_PATH in os.environ: 1183 ↛ 1184line 1183 didn't jump to line 1184, because the condition on line 1183 was never true

1184 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1185 defaultsPaths.extend(externalPaths) 

1186 

1187 # Add the package defaults as a resource 

1188 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1189 forceDirectory=True)) 

1190 return defaultsPaths 

1191 

1192 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1193 """Search the supplied paths, merging the configuration values. 

1194 

1195 The values read will override values currently stored in the object. 

1196 Every file found in the path will be read, such that the earlier 

1197 path entries have higher priority. 

1198 

1199 Parameters 

1200 ---------- 

1201 searchPaths : `list` of `ButlerURI`, `str` 

1202 Paths to search for the supplied configFile. This path 

1203 is the priority order, such that files read from the 

1204 first path entry will be selected over those read from 

1205 a later path. Can contain `str` referring to the local file 

1206 system or a URI string. 

1207 configFile : `ButlerURI` 

1208 File to locate in path. If absolute path it will be read 

1209 directly and the search path will not be used. Can be a URI 

1210 to an explicit resource (which will ignore the search path) 

1211 which is assumed to exist. 

1212 """ 

1213 uri = ButlerURI(configFile) 

1214 if uri.isabs() and uri.exists(): 1214 ↛ 1216line 1214 didn't jump to line 1216, because the condition on line 1214 was never true

1215 # Assume this resource exists 

1216 self._updateWithOtherConfigFile(configFile) 

1217 self.filesRead.append(configFile) 

1218 else: 

1219 # Reverse order so that high priority entries 

1220 # update the object last. 

1221 for pathDir in reversed(searchPaths): 

1222 if isinstance(pathDir, (str, ButlerURI)): 1222 ↛ 1229line 1222 didn't jump to line 1229, because the condition on line 1222 was never false

1223 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1224 file = pathDir.join(configFile) 

1225 if file.exists(): 1225 ↛ 1221line 1225 didn't jump to line 1221, because the condition on line 1225 was never false

1226 self.filesRead.append(file) 

1227 self._updateWithOtherConfigFile(file) 

1228 else: 

1229 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1230 

1231 def _updateWithOtherConfigFile(self, file): 

1232 """Read in some defaults and update. 

1233 

1234 Update the configuration by reading the supplied file as a config 

1235 of this class, and merging such that these values override the 

1236 current values. Contents of the external config are not validated. 

1237 

1238 Parameters 

1239 ---------- 

1240 file : `Config`, `str`, `ButlerURI`, or `dict` 

1241 Entity that can be converted to a `ConfigSubset`. 

1242 """ 

1243 # Use this class to read the defaults so that subsetting can happen 

1244 # correctly. 

1245 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1246 self.update(externalConfig) 

1247 

1248 def validate(self): 

1249 """Check that mandatory keys are present in this configuration. 

1250 

1251 Ignored if ``requiredKeys`` is empty. 

1252 """ 

1253 # Validation 

1254 missing = [k for k in self.requiredKeys if k not in self._data] 

1255 if missing: 1255 ↛ 1256line 1255 didn't jump to line 1256, because the condition on line 1255 was never true

1256 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")