Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from pathlib import Path 

37from yaml.representer import Representer 

38import io 

39from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union 

40 

41from lsst.utils import doImport 

42from ._butlerUri import ButlerURI 

43 

44yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

45 

46 

47# Config module logger 

48log = logging.getLogger(__name__) 

49 

50# PATH-like environment variable to use for defaults. 

51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

52 

53try: 

54 yamlLoader = yaml.CSafeLoader 

55except AttributeError: 

56 # Not all installations have the C library 

57 # (but assume for mypy's sake that they're the same) 

58 yamlLoader = yaml.SafeLoader # type: ignore 

59 

60 

61def _doUpdate(d, u): 

62 if not isinstance(u, collections.abc.Mapping) or \ 62 ↛ 64line 62 didn't jump to line 64, because the condition on line 62 was never true

63 not isinstance(d, collections.abc.MutableMapping): 

64 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

65 for k, v in u.items(): 

66 if isinstance(v, collections.abc.Mapping): 

67 d[k] = _doUpdate(d.get(k, {}), v) 

68 else: 

69 d[k] = v 

70 return d 

71 

72 

73class Loader(yamlLoader): 

74 """YAML Loader that supports file include directives. 

75 

76 Uses ``!include`` directive in a YAML file to point to another 

77 YAML file to be included. The path in the include directive is relative 

78 to the file containing that directive. 

79 

80 storageClasses: !include storageClasses.yaml 

81 

82 Examples 

83 -------- 

84 >>> with open("document.yaml", "r") as f: 

85 data = yaml.load(f, Loader=Loader) 

86 

87 Notes 

88 ----- 

89 See https://davidchall.github.io/yaml-includes.html 

90 """ 

91 

92 def __init__(self, stream): 

93 super().__init__(stream) 

94 # if this is a string and not a stream we may well lack a name 

95 try: 

96 self._root = ButlerURI(stream.name) 

97 except AttributeError: 

98 # No choice but to assume a local filesystem 

99 self._root = ButlerURI("no-file.yaml") 

100 Loader.add_constructor("!include", Loader.include) 

101 

102 def include(self, node): 

103 result: Union[List[Any], Dict[str, Any]] 

104 if isinstance(node, yaml.ScalarNode): 

105 return self.extractFile(self.construct_scalar(node)) 

106 

107 elif isinstance(node, yaml.SequenceNode): 

108 result = [] 

109 for filename in self.construct_sequence(node): 

110 result.append(self.extractFile(filename)) 

111 return result 

112 

113 elif isinstance(node, yaml.MappingNode): 

114 result = {} 

115 for k, v in self.construct_mapping(node).items(): 

116 result[k] = self.extractFile(v) 

117 return result 

118 

119 else: 

120 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

121 raise yaml.constructor.ConstructorError 

122 

123 def extractFile(self, filename): 

124 # It is possible for the !include to point to an explicit URI 

125 # instead of a relative URI, therefore we first see if it is 

126 # scheme-less or not. If it has a scheme we use it directly 

127 # if it is scheme-less we use it relative to the file root. 

128 requesteduri = ButlerURI(filename, forceAbsolute=False) 

129 

130 if requesteduri.scheme: 

131 fileuri = requesteduri 

132 else: 

133 fileuri = self._root.updatedFile(filename) 

134 

135 log.debug("Opening YAML file via !include: %s", fileuri) 

136 

137 # Read all the data from the resource 

138 data = fileuri.read() 

139 

140 # Store the bytes into a BytesIO so we can attach a .name 

141 stream = io.BytesIO(data) 

142 stream.name = fileuri.geturl() 

143 return yaml.load(stream, Loader) 

144 

145 

146class Config(collections.abc.MutableMapping): 

147 r"""Implements a datatype that is used by `Butler` for configuration. 

148 

149 It is essentially a `dict` with key/value pairs, including nested dicts 

150 (as values). In fact, it can be initialized with a `dict`. 

151 This is explained next: 

152 

153 Config extends the `dict` api so that hierarchical values may be accessed 

154 with delimited notation or as a tuple. If a string is given the delimiter 

155 is picked up from the first character in that string. For example, 

156 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

157 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

158 If the first character is alphanumeric, no delimiter will be used. 

159 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

160 Unicode characters can be used as the delimiter for distinctiveness if 

161 required. 

162 

163 If a key in the hierarchy starts with a non-alphanumeric character care 

164 should be used to ensure that either the tuple interface is used or 

165 a distinct delimiter is always given in string form. 

166 

167 Finally, the delimiter can be escaped if it is part of a key and also 

168 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

169 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

170 always better to use a different delimiter in these cases. 

171 

172 Note that adding a multi-level key implicitly creates any nesting levels 

173 that do not exist, but removing multi-level keys does not automatically 

174 remove empty nesting levels. As a result: 

175 

176 >>> c = Config() 

177 >>> c[".a.b"] = 1 

178 >>> del c[".a.b"] 

179 >>> c["a"] 

180 Config({'a': {}}) 

181 

182 Storage formats supported: 

183 

184 - yaml: read and write is supported. 

185 - json: read and write is supported but no ``!include`` directive. 

186 

187 Parameters 

188 ---------- 

189 other : `str` or `Config` or `dict` or `ButlerURI` or `pathlib.Path` 

190 Other source of configuration, can be: 

191 

192 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

193 with ".yaml". 

194 - (`Config`) Copies the other Config's values into this one. 

195 - (`dict`) Copies the values from the dict into this Config. 

196 

197 If `None` is provided an empty `Config` will be created. 

198 """ 

199 

200 _D: str = "→" 

201 """Default internal delimiter to use for components in the hierarchy when 

202 constructing keys for external use (see `Config.names()`).""" 

203 

204 includeKey: ClassVar[str] = "includeConfigs" 

205 """Key used to indicate that another config should be included at this 

206 part of the hierarchy.""" 

207 

208 resourcesPackage: str = "lsst.daf.butler" 

209 """Package to search for default configuration data. The resources 

210 themselves will be within a ``configs`` resource hierarchy.""" 

211 

212 def __init__(self, other=None): 

213 self._data: Dict[str, Any] = {} 

214 self.configFile = None 

215 

216 if other is None: 

217 return 

218 

219 if isinstance(other, Config): 

220 self._data = copy.deepcopy(other._data) 

221 self.configFile = other.configFile 

222 elif isinstance(other, collections.abc.Mapping): 

223 self.update(other) 

224 elif isinstance(other, (str, ButlerURI, Path)): 224 ↛ 231line 224 didn't jump to line 231, because the condition on line 224 was never false

225 # if other is a string, assume it is a file path/URI 

226 self.__initFromUri(other) 

227 self._processExplicitIncludes() 

228 else: 

229 # if the config specified by other could not be recognized raise 

230 # a runtime error. 

231 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

232 

233 def ppprint(self): 

234 """Return config as formatted readable string. 

235 

236 Examples 

237 -------- 

238 use: ``pdb> print(myConfigObject.ppprint())`` 

239 

240 Returns 

241 ------- 

242 s : `str` 

243 A prettyprint formatted string representing the config 

244 """ 

245 return pprint.pformat(self._data, indent=2, width=1) 

246 

247 def __repr__(self): 

248 return f"{type(self).__name__}({self._data!r})" 

249 

250 def __str__(self): 

251 return self.ppprint() 

252 

253 def __len__(self): 

254 return len(self._data) 

255 

256 def __iter__(self): 

257 return iter(self._data) 

258 

259 def copy(self): 

260 return type(self)(self) 

261 

262 @classmethod 

263 def fromString(cls, string: str, format: str = "yaml") -> Config: 

264 """Create a new Config instance from a serialized string. 

265 

266 Parameters 

267 ---------- 

268 string : `str` 

269 String containing content in specified format 

270 format : `str`, optional 

271 Format of the supplied string. Can be ``json`` or ``yaml``. 

272 

273 Returns 

274 ------- 

275 c : `Config` 

276 Newly-constructed Config. 

277 """ 

278 if format == "yaml": 

279 new_config = cls().__initFromYaml(string) 

280 elif format == "json": 

281 new_config = cls().__initFromJson(string) 

282 else: 

283 raise ValueError(f"Unexpected format of string: {format}") 

284 new_config._processExplicitIncludes() 

285 return new_config 

286 

287 @classmethod 

288 def fromYaml(cls, string: str) -> Config: 

289 """Create a new Config instance from a YAML string. 

290 

291 Parameters 

292 ---------- 

293 string : `str` 

294 String containing content in YAML format 

295 

296 Returns 

297 ------- 

298 c : `Config` 

299 Newly-constructed Config. 

300 """ 

301 return cls.fromString(string, format="yaml") 

302 

303 def __initFromUri(self, path: Union[str, ButlerURI, Path]) -> None: 

304 """Load a file from a path or an URI. 

305 

306 Parameters 

307 ---------- 

308 path : `str` 

309 Path or a URI to a persisted config file. 

310 """ 

311 uri = ButlerURI(path) 

312 ext = uri.getExtension() 

313 if ext == ".yaml": 313 ↛ 320line 313 didn't jump to line 320, because the condition on line 313 was never false

314 log.debug("Opening YAML config file: %s", uri.geturl()) 

315 content = uri.read() 

316 # Use a stream so we can name it 

317 stream = io.BytesIO(content) 

318 stream.name = uri.geturl() 

319 self.__initFromYaml(stream) 

320 elif ext == ".json": 

321 log.debug("Opening JSON config file: %s", uri.geturl()) 

322 content = uri.read() 

323 self.__initFromJson(content) 

324 else: 

325 # This URI does not have a valid extension. It might be because 

326 # we ended up with a directory and not a file. Before we complain 

327 # about an extension, do an existence check. No need to do 

328 # the (possibly expensive) existence check in the default code 

329 # path above because we will find out soon enough that the file 

330 # is not there. 

331 if not uri.exists(): 

332 raise FileNotFoundError(f"Config location {uri} does not exist.") 

333 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

334 self.configFile = uri 

335 

336 def __initFromYaml(self, stream): 

337 """Load a YAML config from any readable stream that contains one. 

338 

339 Parameters 

340 ---------- 

341 stream: `IO` or `str` 

342 Stream to pass to the YAML loader. Accepts anything that 

343 `yaml.load` accepts. This can include a string as well as an 

344 IO stream. 

345 

346 Raises 

347 ------ 

348 yaml.YAMLError 

349 If there is an error loading the file. 

350 """ 

351 content = yaml.load(stream, Loader=Loader) 

352 if content is None: 352 ↛ 353line 352 didn't jump to line 353, because the condition on line 352 was never true

353 content = {} 

354 self._data = content 

355 return self 

356 

357 def __initFromJson(self, stream): 

358 """Load a JSON config from any readable stream that contains one. 

359 

360 Parameters 

361 ---------- 

362 stream: `IO` or `str` 

363 Stream to pass to the JSON loader. This can include a string as 

364 well as an IO stream. 

365 

366 Raises 

367 ------ 

368 TypeError: 

369 Raised if there is an error loading the content. 

370 """ 

371 if isinstance(stream, (bytes, str)): 

372 content = json.loads(stream) 

373 else: 

374 content = json.load(stream) 

375 if content is None: 

376 content = {} 

377 self._data = content 

378 return self 

379 

380 def _processExplicitIncludes(self): 

381 """Scan through the configuration searching for the special includes. 

382 

383 Looks for ``includeConfigs`` directive and processes the includes. 

384 """ 

385 # Search paths for config files 

386 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

387 if self.configFile is not None: 387 ↛ 395line 387 didn't jump to line 395, because the condition on line 387 was never false

388 if isinstance(self.configFile, ButlerURI): 388 ↛ 391line 388 didn't jump to line 391, because the condition on line 388 was never false

389 configDir = self.configFile.dirname() 

390 else: 

391 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

392 searchPaths.append(configDir) 

393 

394 # Ensure we know what delimiter to use 

395 names = self.nameTuples() 

396 for path in names: 

397 if path[-1] == self.includeKey: 397 ↛ 399line 397 didn't jump to line 399, because the condition on line 397 was never true

398 

399 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

400 basePath = path[:-1] 

401 

402 # Extract the includes and then delete them from the config 

403 includes = self[path] 

404 del self[path] 

405 

406 # Be consistent and convert to a list 

407 if not isinstance(includes, list): 

408 includes = [includes] 

409 

410 # Read each file assuming it is a reference to a file 

411 # The file can be relative to config file or cwd 

412 # ConfigSubset search paths are not used 

413 subConfigs = [] 

414 for fileName in includes: 

415 # Expand any shell variables -- this could be URI 

416 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

417 found = None 

418 if fileName.isabs(): 

419 found = fileName 

420 else: 

421 for dir in searchPaths: 

422 if isinstance(dir, ButlerURI): 

423 specific = dir.join(fileName.path) 

424 # Remote resource check might be expensive 

425 if specific.exists(): 

426 found = specific 

427 else: 

428 log.warning("Do not understand search path entry '%s' of type %s", 

429 dir, type(dir).__name__) 

430 if not found: 

431 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

432 

433 # Read the referenced Config as a Config 

434 subConfigs.append(type(self)(found)) 

435 

436 # Now we need to merge these sub configs with the current 

437 # information that was present in this node in the config 

438 # tree with precedence given to the explicit values 

439 newConfig = subConfigs.pop(0) 

440 for sc in subConfigs: 

441 newConfig.update(sc) 

442 

443 # Explicit values take precedence 

444 if not basePath: 

445 # This is an include at the root config 

446 newConfig.update(self) 

447 # Replace the current config 

448 self._data = newConfig._data 

449 else: 

450 newConfig.update(self[basePath]) 

451 # And reattach to the base config 

452 self[basePath] = newConfig 

453 

454 @staticmethod 

455 def _splitIntoKeys(key): 

456 r"""Split the argument for get/set/in into a hierarchical list. 

457 

458 Parameters 

459 ---------- 

460 key : `str` or iterable 

461 Argument given to get/set/in. If an iterable is provided it will 

462 be converted to a list. If the first character of the string 

463 is not an alphanumeric character then it will be used as the 

464 delimiter for the purposes of splitting the remainder of the 

465 string. If the delimiter is also in one of the keys then it 

466 can be escaped using ``\``. There is no default delimiter. 

467 

468 Returns 

469 ------- 

470 keys : `list` 

471 Hierarchical keys as a `list`. 

472 """ 

473 if isinstance(key, str): 

474 if not key[0].isalnum(): 474 ↛ 475line 474 didn't jump to line 475, because the condition on line 474 was never true

475 d = key[0] 

476 key = key[1:] 

477 else: 

478 return [key, ] 

479 escaped = f"\\{d}" 

480 temp = None 

481 if escaped in key: 

482 # Complain at the attempt to escape the escape 

483 doubled = fr"\{escaped}" 

484 if doubled in key: 

485 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

486 " is not yet supported.") 

487 # Replace with a character that won't be in the string 

488 temp = "\r" 

489 if temp in key or d == temp: 

490 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

491 " delimiter if escaping the delimiter") 

492 key = key.replace(escaped, temp) 

493 hierarchy = key.split(d) 

494 if temp: 

495 hierarchy = [h.replace(temp, d) for h in hierarchy] 

496 return hierarchy 

497 elif isinstance(key, collections.abc.Iterable): 497 ↛ 501line 497 didn't jump to line 501, because the condition on line 497 was never false

498 return list(key) 

499 else: 

500 # Not sure what this is so try it anyway 

501 return [key, ] 

502 

503 def _getKeyHierarchy(self, name): 

504 """Retrieve the key hierarchy for accessing the Config. 

505 

506 Parameters 

507 ---------- 

508 name : `str` or `tuple` 

509 Delimited string or `tuple` of hierarchical keys. 

510 

511 Returns 

512 ------- 

513 hierarchy : `list` of `str` 

514 Hierarchy to use as a `list`. If the name is available directly 

515 as a key in the Config it will be used regardless of the presence 

516 of any nominal delimiter. 

517 """ 

518 if name in self._data: 

519 keys = [name, ] 

520 else: 

521 keys = self._splitIntoKeys(name) 

522 return keys 

523 

524 def _findInHierarchy(self, keys, create=False): 

525 """Look for hierarchy of keys in Config. 

526 

527 Parameters 

528 ---------- 

529 keys : `list` or `tuple` 

530 Keys to search in hierarchy. 

531 create : `bool`, optional 

532 If `True`, if a part of the hierarchy does not exist, insert an 

533 empty `dict` into the hierarchy. 

534 

535 Returns 

536 ------- 

537 hierarchy : `list` 

538 List of the value corresponding to each key in the supplied 

539 hierarchy. Only keys that exist in the hierarchy will have 

540 a value. 

541 complete : `bool` 

542 `True` if the full hierarchy exists and the final element 

543 in ``hierarchy`` is the value of relevant value. 

544 """ 

545 d = self._data 

546 

547 def checkNextItem(k, d, create): 

548 """See if k is in d and if it is return the new child.""" 

549 nextVal = None 

550 isThere = False 

551 if d is None: 551 ↛ 553line 551 didn't jump to line 553, because the condition on line 551 was never true

552 # We have gone past the end of the hierarchy 

553 pass 

554 elif isinstance(d, collections.abc.Sequence): 554 ↛ 559line 554 didn't jump to line 559, because the condition on line 554 was never true

555 # Check sequence first because for lists 

556 # __contains__ checks whether value is found in list 

557 # not whether the index exists in list. When we traverse 

558 # the hierarchy we are interested in the index. 

559 try: 

560 nextVal = d[int(k)] 

561 isThere = True 

562 except IndexError: 

563 pass 

564 except ValueError: 

565 isThere = k in d 

566 elif k in d: 

567 nextVal = d[k] 

568 isThere = True 

569 elif create: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true

570 d[k] = {} 

571 nextVal = d[k] 

572 isThere = True 

573 return nextVal, isThere 

574 

575 hierarchy = [] 

576 complete = True 

577 for k in keys: 

578 d, isThere = checkNextItem(k, d, create) 

579 if isThere: 

580 hierarchy.append(d) 

581 else: 

582 complete = False 

583 break 

584 

585 return hierarchy, complete 

586 

587 def __getitem__(self, name): 

588 # Override the split for the simple case where there is an exact 

589 # match. This allows `Config.items()` to work via a simple 

590 # __iter__ implementation that returns top level keys of 

591 # self._data. 

592 keys = self._getKeyHierarchy(name) 

593 

594 hierarchy, complete = self._findInHierarchy(keys) 

595 if not complete: 595 ↛ 596line 595 didn't jump to line 596, because the condition on line 595 was never true

596 raise KeyError(f"{name} not found") 

597 data = hierarchy[-1] 

598 

599 if isinstance(data, collections.abc.Mapping): 

600 data = Config(data) 

601 # Ensure that child configs inherit the parent internal delimiter 

602 if self._D != Config._D: 602 ↛ 603line 602 didn't jump to line 603, because the condition on line 602 was never true

603 data._D = self._D 

604 return data 

605 

606 def __setitem__(self, name, value): 

607 keys = self._getKeyHierarchy(name) 

608 last = keys.pop() 

609 if isinstance(value, Config): 

610 value = copy.deepcopy(value._data) 

611 

612 hierarchy, complete = self._findInHierarchy(keys, create=True) 

613 if hierarchy: 

614 data = hierarchy[-1] 

615 else: 

616 data = self._data 

617 

618 try: 

619 data[last] = value 

620 except TypeError: 

621 data[int(last)] = value 

622 

623 def __contains__(self, key): 

624 keys = self._getKeyHierarchy(key) 

625 hierarchy, complete = self._findInHierarchy(keys) 

626 return complete 

627 

628 def __delitem__(self, key): 

629 keys = self._getKeyHierarchy(key) 

630 last = keys.pop() 

631 hierarchy, complete = self._findInHierarchy(keys) 

632 if complete: 632 ↛ 639line 632 didn't jump to line 639, because the condition on line 632 was never false

633 if hierarchy: 633 ↛ 634line 633 didn't jump to line 634, because the condition on line 633 was never true

634 data = hierarchy[-1] 

635 else: 

636 data = self._data 

637 del data[last] 

638 else: 

639 raise KeyError(f"{key} not found in Config") 

640 

641 def update(self, other): 

642 """Update config from other `Config` or `dict`. 

643 

644 Like `dict.update()`, but will add or modify keys in nested dicts, 

645 instead of overwriting the nested dict entirely. 

646 

647 Parameters 

648 ---------- 

649 other : `dict` or `Config` 

650 Source of configuration: 

651 

652 Examples 

653 -------- 

654 >>> c = Config({"a": {"b": 1}}) 

655 >>> c.update({"a": {"c": 2}}) 

656 >>> print(c) 

657 {'a': {'b': 1, 'c': 2}} 

658 

659 >>> foo = {"a": {"b": 1}} 

660 >>> foo.update({"a": {"c": 2}}) 

661 >>> print(foo) 

662 {'a': {'c': 2}} 

663 """ 

664 _doUpdate(self._data, other) 

665 

666 def merge(self, other): 

667 """Merge another Config into this one. 

668 

669 Like `Config.update()`, but will add keys & values from other that 

670 DO NOT EXIST in self. 

671 

672 Keys and values that already exist in self will NOT be overwritten. 

673 

674 Parameters 

675 ---------- 

676 other : `dict` or `Config` 

677 Source of configuration: 

678 """ 

679 if not isinstance(other, collections.abc.Mapping): 

680 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}") 

681 

682 # Convert the supplied mapping to a Config for consistency 

683 # This will do a deepcopy if it is already a Config 

684 otherCopy = Config(other) 

685 otherCopy.update(self) 

686 self._data = otherCopy._data 

687 

688 def nameTuples(self, topLevelOnly=False): 

689 """Get tuples representing the name hierarchies of all keys. 

690 

691 The tuples returned from this method are guaranteed to be usable 

692 to access items in the configuration object. 

693 

694 Parameters 

695 ---------- 

696 topLevelOnly : `bool`, optional 

697 If False, the default, a full hierarchy of names is returned. 

698 If True, only the top level are returned. 

699 

700 Returns 

701 ------- 

702 names : `list` of `tuple` of `str` 

703 List of all names present in the `Config` where each element 

704 in the list is a `tuple` of strings representing the hierarchy. 

705 """ 

706 if topLevelOnly: 706 ↛ 707line 706 didn't jump to line 707, because the condition on line 706 was never true

707 return list((k,) for k in self) 

708 

709 def getKeysAsTuples(d, keys, base): 

710 if isinstance(d, collections.abc.Sequence): 

711 theseKeys = range(len(d)) 

712 else: 

713 theseKeys = d.keys() 

714 for key in theseKeys: 

715 val = d[key] 

716 levelKey = base + (key,) if base is not None else (key,) 

717 keys.append(levelKey) 

718 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

719 and not isinstance(val, str): 

720 getKeysAsTuples(val, keys, levelKey) 

721 keys: List[Tuple[str, ...]] = [] 

722 getKeysAsTuples(self._data, keys, None) 

723 return keys 

724 

725 def names(self, topLevelOnly=False, delimiter=None): 

726 """Get a delimited name of all the keys in the hierarchy. 

727 

728 The values returned from this method are guaranteed to be usable 

729 to access items in the configuration object. 

730 

731 Parameters 

732 ---------- 

733 topLevelOnly : `bool`, optional 

734 If False, the default, a full hierarchy of names is returned. 

735 If True, only the top level are returned. 

736 delimiter : `str`, optional 

737 Delimiter to use when forming the keys. If the delimiter is 

738 present in any of the keys, it will be escaped in the returned 

739 names. If `None` given a delimiter will be automatically provided. 

740 The delimiter can not be alphanumeric. 

741 

742 Returns 

743 ------- 

744 names : `list` of `str` 

745 List of all names present in the `Config`. 

746 

747 Notes 

748 ----- 

749 This is different than the built-in method `dict.keys`, which will 

750 return only the first level keys. 

751 

752 Raises 

753 ------ 

754 ValueError: 

755 The supplied delimiter is alphanumeric. 

756 """ 

757 if topLevelOnly: 

758 return list(self.keys()) 

759 

760 # Get all the tuples of hierarchical keys 

761 nameTuples = self.nameTuples() 

762 

763 if delimiter is not None and delimiter.isalnum(): 

764 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

765 

766 if delimiter is None: 

767 # Start with something, and ensure it does not need to be 

768 # escaped (it is much easier to understand if not escaped) 

769 delimiter = self._D 

770 

771 # Form big string for easy check of delimiter clash 

772 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

773 

774 # Try a delimiter and keep trying until we get something that 

775 # works. 

776 ntries = 0 

777 while delimiter in combined: 

778 log.debug("Delimiter '%s' could not be used. Trying another.", delimiter) 

779 ntries += 1 

780 

781 if ntries > 100: 

782 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

783 

784 # try another one 

785 while True: 

786 delimiter = chr(ord(delimiter)+1) 

787 if not delimiter.isalnum(): 

788 break 

789 

790 log.debug("Using delimiter %r", delimiter) 

791 

792 # Form the keys, escaping the delimiter if necessary 

793 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

794 for k in nameTuples] 

795 return strings 

796 

797 def asArray(self, name): 

798 """Get a value as an array. 

799 

800 May contain one or more elements. 

801 

802 Parameters 

803 ---------- 

804 name : `str` 

805 Key to use to retrieve value. 

806 

807 Returns 

808 ------- 

809 array : `collections.abc.Sequence` 

810 The value corresponding to name, but guaranteed to be returned 

811 as a list with at least one element. If the value is a 

812 `~collections.abc.Sequence` (and not a `str`) the value itself 

813 will be returned, else the value will be the first element. 

814 """ 

815 val = self.get(name) 

816 if isinstance(val, str): 

817 val = [val] 

818 elif not isinstance(val, collections.abc.Sequence): 

819 val = [val] 

820 return val 

821 

822 def __eq__(self, other): 

823 if isinstance(other, Config): 

824 other = other._data 

825 return self._data == other 

826 

827 def __ne__(self, other): 

828 if isinstance(other, Config): 

829 other = other._data 

830 return self._data != other 

831 

832 ####### 

833 # i/o # 

834 

835 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

836 """Write the config to an output stream. 

837 

838 Parameters 

839 ---------- 

840 output : `IO`, optional 

841 The stream to use for output. If `None` the serialized content 

842 will be returned. 

843 format : `str`, optional 

844 The format to use for the output. Can be "yaml" or "json". 

845 

846 Returns 

847 ------- 

848 serialized : `str` or `None` 

849 If a stream was given the stream will be used and the return 

850 value will be `None`. If the stream was `None` the 

851 serialization will be returned as a string. 

852 """ 

853 if format == "yaml": 

854 return yaml.safe_dump(self._data, output, default_flow_style=False) 

855 elif format == "json": 

856 if output is not None: 

857 json.dump(self._data, output, ensure_ascii=False) 

858 return None 

859 else: 

860 return json.dumps(self._data, ensure_ascii=False) 

861 raise ValueError(f"Unsupported format for Config serialization: {format}") 

862 

863 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

864 defaultFileName: str = "butler.yaml", 

865 overwrite: bool = True) -> None: 

866 """Write the config to location pointed to by given URI. 

867 

868 Currently supports 's3' and 'file' URI schemes. 

869 

870 Parameters 

871 ---------- 

872 uri: `str` or `ButlerURI` 

873 URI of location where the Config will be written. 

874 updateFile : bool, optional 

875 If True and uri does not end on a filename with extension, will 

876 append `defaultFileName` to the target uri. True by default. 

877 defaultFileName : bool, optional 

878 The file name that will be appended to target uri if updateFile is 

879 True and uri does not end on a file with an extension. 

880 overwrite : bool, optional 

881 If True the configuration will be written even if it already 

882 exists at that location. 

883 """ 

884 # Make local copy of URI or create new one 

885 uri = ButlerURI(uri) 

886 

887 if updateFile and not uri.getExtension(): 

888 uri = uri.updatedFile(defaultFileName) 

889 

890 # Try to work out the format from the extension 

891 ext = uri.getExtension() 

892 format = ext[1:].lower() 

893 

894 output = self.dump(format=format) 

895 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

896 uri.write(output.encode(), overwrite=overwrite) 

897 self.configFile = uri 

898 

899 @staticmethod 

900 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True, toMerge=None): 

901 """Update specific config parameters. 

902 

903 Allows for named parameters to be set to new values in bulk, and 

904 for other values to be set by copying from a reference config. 

905 

906 Assumes that the supplied config is compatible with ``configType`` 

907 and will attach the updated values to the supplied config by 

908 looking for the related component key. It is assumed that 

909 ``config`` and ``full`` are from the same part of the 

910 configuration hierarchy. 

911 

912 Parameters 

913 ---------- 

914 configType : `ConfigSubset` 

915 Config type to use to extract relevant items from ``config``. 

916 config : `Config` 

917 A `Config` to update. Only the subset understood by 

918 the supplied `ConfigSubset` will be modified. Default values 

919 will not be inserted and the content will not be validated 

920 since mandatory keys are allowed to be missing until 

921 populated later by merging. 

922 full : `Config` 

923 A complete config with all defaults expanded that can be 

924 converted to a ``configType``. Read-only and will not be 

925 modified by this method. Values are read from here if 

926 ``toCopy`` is defined. 

927 

928 Repository-specific options that should not be obtained 

929 from defaults when Butler instances are constructed 

930 should be copied from ``full`` to ``config``. 

931 toUpdate : `dict`, optional 

932 A `dict` defining the keys to update and the new value to use. 

933 The keys and values can be any supported by `Config` 

934 assignment. 

935 toCopy : `tuple`, optional 

936 `tuple` of keys whose values should be copied from ``full`` 

937 into ``config``. 

938 overwrite : `bool`, optional 

939 If `False`, do not modify a value in ``config`` if the key 

940 already exists. Default is always to overwrite. 

941 toMerge : `tuple`, optional 

942 Keys to merge content from full to config without overwriting 

943 pre-existing values. Only works if the key refers to a hierarchy. 

944 The ``overwrite`` flag is ignored. 

945 

946 Raises 

947 ------ 

948 ValueError 

949 Neither ``toUpdate``, ``toCopy`` nor ``toMerge`` were defined. 

950 """ 

951 if toUpdate is None and toCopy is None and toMerge is None: 

952 raise ValueError("At least one of toUpdate, toCopy, or toMerge parameters must be set.") 

953 

954 # If this is a parent configuration then we need to ensure that 

955 # the supplied config has the relevant component key in it. 

956 # If this is a parent configuration we add in the stub entry 

957 # so that the ConfigSubset constructor will do the right thing. 

958 # We check full for this since that is guaranteed to be complete. 

959 if configType.component in full and configType.component not in config: 

960 config[configType.component] = {} 

961 

962 # Extract the part of the config we wish to update 

963 localConfig = configType(config, mergeDefaults=False, validate=False) 

964 

965 if toUpdate: 

966 for key, value in toUpdate.items(): 

967 if key in localConfig and not overwrite: 

968 log.debug("Not overriding key '%s' with value '%s' in config %s", 

969 key, value, localConfig.__class__.__name__) 

970 else: 

971 localConfig[key] = value 

972 

973 if toCopy or toMerge: 

974 localFullConfig = configType(full, mergeDefaults=False) 

975 

976 if toCopy: 

977 for key in toCopy: 

978 if key in localConfig and not overwrite: 

979 log.debug("Not overriding key '%s' from defaults in config %s", 

980 key, localConfig.__class__.__name__) 

981 else: 

982 localConfig[key] = localFullConfig[key] 

983 if toMerge: 

984 for key in toMerge: 

985 if key in localConfig: 

986 # Get the node from the config to do the merge 

987 # but then have to reattach to the config. 

988 subset = localConfig[key] 

989 subset.merge(localFullConfig[key]) 

990 localConfig[key] = subset 

991 else: 

992 localConfig[key] = localFullConfig[key] 

993 

994 # Reattach to parent if this is a child config 

995 if configType.component in config: 

996 config[configType.component] = localConfig 

997 else: 

998 config.update(localConfig) 

999 

1000 def toDict(self): 

1001 """Convert a `Config` to a standalone hierarchical `dict`. 

1002 

1003 Returns 

1004 ------- 

1005 d : `dict` 

1006 The standalone hierarchical `dict` with any `Config` classes 

1007 in the hierarchy converted to `dict`. 

1008 

1009 Notes 

1010 ----- 

1011 This can be useful when passing a Config to some code that 

1012 expects native Python types. 

1013 """ 

1014 output = copy.deepcopy(self._data) 

1015 for k, v in output.items(): 

1016 if isinstance(v, Config): 

1017 v = v.toDict() 

1018 output[k] = v 

1019 return output 

1020 

1021 

1022class ConfigSubset(Config): 

1023 """Config representing a subset of a more general configuration. 

1024 

1025 Subclasses define their own component and when given a configuration 

1026 that includes that component, the resulting configuration only includes 

1027 the subset. For example, your config might contain ``dimensions`` if it's 

1028 part of a global config and that subset will be stored. If ``dimensions`` 

1029 can not be found it is assumed that the entire contents of the 

1030 configuration should be used. 

1031 

1032 Default values are read from the environment or supplied search paths 

1033 using the default configuration file name specified in the subclass. 

1034 This allows a configuration class to be instantiated without any 

1035 additional arguments. 

1036 

1037 Additional validation can be specified to check for keys that are mandatory 

1038 in the configuration. 

1039 

1040 Parameters 

1041 ---------- 

1042 other : `Config` or `str` or `dict` 

1043 Argument specifying the configuration information as understood 

1044 by `Config` 

1045 validate : `bool`, optional 

1046 If `True` required keys will be checked to ensure configuration 

1047 consistency. 

1048 mergeDefaults : `bool`, optional 

1049 If `True` defaults will be read and the supplied config will 

1050 be combined with the defaults, with the supplied valiues taking 

1051 precedence. 

1052 searchPaths : `list` or `tuple`, optional 

1053 Explicit additional paths to search for defaults. They should 

1054 be supplied in priority order. These paths have higher priority 

1055 than those read from the environment in 

1056 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1057 the local file system or URIs, `ButlerURI`. 

1058 """ 

1059 

1060 component: ClassVar[Optional[str]] = None 

1061 """Component to use from supplied config. Can be None. If specified the 

1062 key is not required. Can be a full dot-separated path to a component. 

1063 """ 

1064 

1065 requiredKeys: ClassVar[Sequence[str]] = () 

1066 """Keys that are required to be specified in the configuration. 

1067 """ 

1068 

1069 defaultConfigFile: ClassVar[Optional[str]] = None 

1070 """Name of the file containing defaults for this config class. 

1071 """ 

1072 

1073 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1074 

1075 # Create a blank object to receive the defaults 

1076 # Once we have the defaults we then update with the external values 

1077 super().__init__() 

1078 

1079 # Create a standard Config rather than subset 

1080 externalConfig = Config(other) 

1081 

1082 # Select the part we need from it 

1083 # To simplify the use of !include we also check for the existence of 

1084 # component.component (since the included files can themselves 

1085 # include the component name) 

1086 if self.component is not None: 1086 ↛ 1095line 1086 didn't jump to line 1095, because the condition on line 1086 was never false

1087 doubled = (self.component, self.component) 

1088 # Must check for double depth first 

1089 if doubled in externalConfig: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true

1090 externalConfig = externalConfig[doubled] 

1091 elif self.component in externalConfig: 

1092 externalConfig._data = externalConfig._data[self.component] 

1093 

1094 # Default files read to create this configuration 

1095 self.filesRead = [] 

1096 

1097 # Assume we are not looking up child configurations 

1098 containerKey = None 

1099 

1100 # Sometimes we do not want to merge with defaults. 

1101 if mergeDefaults: 

1102 

1103 # Supplied search paths have highest priority 

1104 fullSearchPath = [] 

1105 if searchPaths: 1105 ↛ 1106line 1105 didn't jump to line 1106, because the condition on line 1105 was never true

1106 fullSearchPath.extend(searchPaths) 

1107 

1108 # Read default paths from enviroment 

1109 fullSearchPath.extend(self.defaultSearchPaths()) 

1110 

1111 # There are two places to find defaults for this particular config 

1112 # - The "defaultConfigFile" defined in the subclass 

1113 # - The class specified in the "cls" element in the config. 

1114 # Read cls after merging in case it changes. 

1115 if self.defaultConfigFile is not None: 1115 ↛ 1120line 1115 didn't jump to line 1120, because the condition on line 1115 was never false

1116 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1117 

1118 # Can have a class specification in the external config (priority) 

1119 # or from the defaults. 

1120 pytype = None 

1121 if "cls" in externalConfig: 1121 ↛ 1122line 1121 didn't jump to line 1122, because the condition on line 1121 was never true

1122 pytype = externalConfig["cls"] 

1123 elif "cls" in self: 1123 ↛ 1124line 1123 didn't jump to line 1124, because the condition on line 1123 was never true

1124 pytype = self["cls"] 

1125 

1126 if pytype is not None: 1126 ↛ 1127line 1126 didn't jump to line 1127, because the condition on line 1126 was never true

1127 try: 

1128 cls = doImport(pytype) 

1129 except ImportError as e: 

1130 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1131 defaultsFile = cls.defaultConfigFile 

1132 if defaultsFile is not None: 

1133 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1134 

1135 # Get the container key in case we need it 

1136 try: 

1137 containerKey = cls.containerKey 

1138 except AttributeError: 

1139 pass 

1140 

1141 # Now update this object with the external values so that the external 

1142 # values always override the defaults 

1143 self.update(externalConfig) 

1144 

1145 # If this configuration has child configurations of the same 

1146 # config class, we need to expand those defaults as well. 

1147 

1148 if mergeDefaults and containerKey is not None and containerKey in self: 1148 ↛ 1149line 1148 didn't jump to line 1149, because the condition on line 1148 was never true

1149 for idx, subConfig in enumerate(self[containerKey]): 

1150 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1151 mergeDefaults=mergeDefaults, 

1152 searchPaths=searchPaths) 

1153 

1154 if validate: 

1155 self.validate() 

1156 

1157 @classmethod 

1158 def defaultSearchPaths(cls): 

1159 """Read environment to determine search paths to use. 

1160 

1161 Global defaults, at lowest priority, are found in the ``config`` 

1162 directory of the butler source tree. Additional defaults can be 

1163 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1164 which is a PATH-like variable where paths at the front of the list 

1165 have priority over those later. 

1166 

1167 Returns 

1168 ------- 

1169 paths : `list` 

1170 Returns a list of paths to search. The returned order is in 

1171 priority with the highest priority paths first. The butler config 

1172 configuration resources will not be included here but will 

1173 always be searched last. 

1174 

1175 Notes 

1176 ----- 

1177 The environment variable is split on the standard ``:`` path separator. 

1178 This currently makes it incompatible with usage of URIs. 

1179 """ 

1180 # We can pick up defaults from multiple search paths 

1181 # We fill defaults by using the butler config path and then 

1182 # the config path environment variable in reverse order. 

1183 defaultsPaths: List[Union[str, ButlerURI]] = [] 

1184 

1185 if CONFIG_PATH in os.environ: 1185 ↛ 1186line 1185 didn't jump to line 1186, because the condition on line 1185 was never true

1186 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1187 defaultsPaths.extend(externalPaths) 

1188 

1189 # Add the package defaults as a resource 

1190 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1191 forceDirectory=True)) 

1192 return defaultsPaths 

1193 

1194 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1195 """Search the supplied paths, merging the configuration values. 

1196 

1197 The values read will override values currently stored in the object. 

1198 Every file found in the path will be read, such that the earlier 

1199 path entries have higher priority. 

1200 

1201 Parameters 

1202 ---------- 

1203 searchPaths : `list` of `ButlerURI`, `str` 

1204 Paths to search for the supplied configFile. This path 

1205 is the priority order, such that files read from the 

1206 first path entry will be selected over those read from 

1207 a later path. Can contain `str` referring to the local file 

1208 system or a URI string. 

1209 configFile : `ButlerURI` 

1210 File to locate in path. If absolute path it will be read 

1211 directly and the search path will not be used. Can be a URI 

1212 to an explicit resource (which will ignore the search path) 

1213 which is assumed to exist. 

1214 """ 

1215 uri = ButlerURI(configFile) 

1216 if uri.isabs() and uri.exists(): 1216 ↛ 1218line 1216 didn't jump to line 1218, because the condition on line 1216 was never true

1217 # Assume this resource exists 

1218 self._updateWithOtherConfigFile(configFile) 

1219 self.filesRead.append(configFile) 

1220 else: 

1221 # Reverse order so that high priority entries 

1222 # update the object last. 

1223 for pathDir in reversed(searchPaths): 

1224 if isinstance(pathDir, (str, ButlerURI)): 1224 ↛ 1231line 1224 didn't jump to line 1231, because the condition on line 1224 was never false

1225 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1226 file = pathDir.join(configFile) 

1227 if file.exists(): 1227 ↛ 1223line 1227 didn't jump to line 1223, because the condition on line 1227 was never false

1228 self.filesRead.append(file) 

1229 self._updateWithOtherConfigFile(file) 

1230 else: 

1231 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1232 

1233 def _updateWithOtherConfigFile(self, file): 

1234 """Read in some defaults and update. 

1235 

1236 Update the configuration by reading the supplied file as a config 

1237 of this class, and merging such that these values override the 

1238 current values. Contents of the external config are not validated. 

1239 

1240 Parameters 

1241 ---------- 

1242 file : `Config`, `str`, `ButlerURI`, or `dict` 

1243 Entity that can be converted to a `ConfigSubset`. 

1244 """ 

1245 # Use this class to read the defaults so that subsetting can happen 

1246 # correctly. 

1247 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1248 self.update(externalConfig) 

1249 

1250 def validate(self): 

1251 """Check that mandatory keys are present in this configuration. 

1252 

1253 Ignored if ``requiredKeys`` is empty. 

1254 """ 

1255 # Validation 

1256 missing = [k for k in self.requiredKeys if k not in self._data] 

1257 if missing: 1257 ↛ 1258line 1257 didn't jump to line 1258, because the condition on line 1257 was never true

1258 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")