Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from pathlib import Path 

37from yaml.representer import Representer 

38import io 

39from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union 

40 

41from lsst.utils import doImport 

42from ._butlerUri import ButlerURI 

43 

44yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

45 

46 

47# Config module logger 

48log = logging.getLogger(__name__) 

49 

50# PATH-like environment variable to use for defaults. 

51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

52 

53try: 

54 yamlLoader = yaml.CSafeLoader 

55except AttributeError: 

56 # Not all installations have the C library 

57 # (but assume for mypy's sake that they're the same) 

58 yamlLoader = yaml.SafeLoader # type: ignore 

59 

60 

61class Loader(yamlLoader): 

62 """YAML Loader that supports file include directives 

63 

64 Uses ``!include`` directive in a YAML file to point to another 

65 YAML file to be included. The path in the include directive is relative 

66 to the file containing that directive. 

67 

68 storageClasses: !include storageClasses.yaml 

69 

70 Examples 

71 -------- 

72 >>> with open("document.yaml", "r") as f: 

73 data = yaml.load(f, Loader=Loader) 

74 

75 Notes 

76 ----- 

77 See https://davidchall.github.io/yaml-includes.html 

78 """ 

79 

80 def __init__(self, stream): 

81 super().__init__(stream) 

82 # if this is a string and not a stream we may well lack a name 

83 try: 

84 self._root = ButlerURI(stream.name) 

85 except AttributeError: 

86 # No choice but to assume a local filesystem 

87 self._root = ButlerURI("no-file.yaml") 

88 Loader.add_constructor("!include", Loader.include) 

89 

90 def include(self, node): 

91 result: Union[List[Any], Dict[str, Any]] 

92 if isinstance(node, yaml.ScalarNode): 

93 return self.extractFile(self.construct_scalar(node)) 

94 

95 elif isinstance(node, yaml.SequenceNode): 

96 result = [] 

97 for filename in self.construct_sequence(node): 

98 result.append(self.extractFile(filename)) 

99 return result 

100 

101 elif isinstance(node, yaml.MappingNode): 

102 result = {} 

103 for k, v in self.construct_mapping(node).items(): 

104 result[k] = self.extractFile(v) 

105 return result 

106 

107 else: 

108 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

109 raise yaml.constructor.ConstructorError 

110 

111 def extractFile(self, filename): 

112 # It is possible for the !include to point to an explicit URI 

113 # instead of a relative URI, therefore we first see if it is 

114 # scheme-less or not. If it has a scheme we use it directly 

115 # if it is scheme-less we use it relative to the file root. 

116 requesteduri = ButlerURI(filename, forceAbsolute=False) 

117 

118 if requesteduri.scheme: 

119 fileuri = requesteduri 

120 else: 

121 fileuri = copy.copy(self._root) 

122 fileuri.updateFile(filename) 

123 

124 log.debug("Opening YAML file via !include: %s", fileuri) 

125 

126 # Read all the data from the resource 

127 data = fileuri.read() 

128 

129 # Store the bytes into a BytesIO so we can attach a .name 

130 stream = io.BytesIO(data) 

131 stream.name = fileuri.geturl() 

132 return yaml.load(stream, Loader) 

133 

134 

135class Config(collections.abc.MutableMapping): 

136 r"""Implements a datatype that is used by `Butler` for configuration 

137 parameters. 

138 

139 It is essentially a `dict` with key/value pairs, including nested dicts 

140 (as values). In fact, it can be initialized with a `dict`. 

141 This is explained next: 

142 

143 Config extends the `dict` api so that hierarchical values may be accessed 

144 with delimited notation or as a tuple. If a string is given the delimiter 

145 is picked up from the first character in that string. For example, 

146 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

147 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

148 If the first character is alphanumeric, no delimiter will be used. 

149 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

150 Unicode characters can be used as the delimiter for distinctiveness if 

151 required. 

152 

153 If a key in the hierarchy starts with a non-alphanumeric character care 

154 should be used to ensure that either the tuple interface is used or 

155 a distinct delimiter is always given in string form. 

156 

157 Finally, the delimiter can be escaped if it is part of a key and also 

158 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

159 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

160 always better to use a different delimiter in these cases. 

161 

162 Note that adding a multi-level key implicitly creates any nesting levels 

163 that do not exist, but removing multi-level keys does not automatically 

164 remove empty nesting levels. As a result: 

165 

166 >>> c = Config() 

167 >>> c[".a.b"] = 1 

168 >>> del c[".a.b"] 

169 >>> c["a"] 

170 Config({'a': {}}) 

171 

172 Storage formats supported: 

173 

174 - yaml: read and write is supported. 

175 - json: read and write is supported but no ``!include`` directive. 

176 

177 Parameters 

178 ---------- 

179 other : `str` or `Config` or `dict` or `ButlerURI` or `pathlib.Path` 

180 Other source of configuration, can be: 

181 

182 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

183 with ".yaml". 

184 - (`Config`) Copies the other Config's values into this one. 

185 - (`dict`) Copies the values from the dict into this Config. 

186 

187 If `None` is provided an empty `Config` will be created. 

188 """ 

189 

190 _D: str = "→" 

191 """Default internal delimiter to use for components in the hierarchy when 

192 constructing keys for external use (see `Config.names()`).""" 

193 

194 includeKey: ClassVar[str] = "includeConfigs" 

195 """Key used to indicate that another config should be included at this 

196 part of the hierarchy.""" 

197 

198 resourcesPackage: str = "lsst.daf.butler" 

199 """Package to search for default configuration data. The resources 

200 themselves will be within a ``configs`` resource hierarchy.""" 

201 

202 def __init__(self, other=None): 

203 self._data: Dict[str, Any] = {} 

204 self.configFile = None 

205 

206 if other is None: 

207 return 

208 

209 if isinstance(other, Config): 

210 self._data = copy.deepcopy(other._data) 

211 self.configFile = other.configFile 

212 elif isinstance(other, collections.abc.Mapping): 

213 self.update(other) 

214 elif isinstance(other, (str, ButlerURI, Path)): 214 ↛ 221line 214 didn't jump to line 221, because the condition on line 214 was never false

215 # if other is a string, assume it is a file path/URI 

216 self.__initFromUri(other) 

217 self._processExplicitIncludes() 

218 else: 

219 # if the config specified by other could not be recognized raise 

220 # a runtime error. 

221 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

222 

223 def ppprint(self): 

224 """helper function for debugging, prints a config out in a readable 

225 way in the debugger. 

226 

227 use: pdb> print(myConfigObject.ppprint()) 

228 

229 Returns 

230 ------- 

231 s : `str` 

232 A prettyprint formatted string representing the config 

233 """ 

234 return pprint.pformat(self._data, indent=2, width=1) 

235 

236 def __repr__(self): 

237 return f"{type(self).__name__}({self._data!r})" 

238 

239 def __str__(self): 

240 return self.ppprint() 

241 

242 def __len__(self): 

243 return len(self._data) 

244 

245 def __iter__(self): 

246 return iter(self._data) 

247 

248 def copy(self): 

249 return type(self)(self) 

250 

251 @classmethod 

252 def fromString(cls, string: str, format: str = "yaml") -> Config: 

253 """Create a new Config instance from a serialized string. 

254 

255 Parameters 

256 ---------- 

257 string : `str` 

258 String containing content in specified format 

259 format : `str`, optional 

260 Format of the supplied string. Can be ``json`` or ``yaml``. 

261 

262 Returns 

263 ------- 

264 c : `Config` 

265 Newly-constructed Config. 

266 """ 

267 if format == "yaml": 

268 new_config = cls().__initFromYaml(string) 

269 elif format == "json": 

270 new_config = cls().__initFromJson(string) 

271 else: 

272 raise ValueError(f"Unexpected format of string: {format}") 

273 new_config._processExplicitIncludes() 

274 return new_config 

275 

276 @classmethod 

277 def fromYaml(cls, string: str) -> Config: 

278 """Create a new Config instance from a YAML string. 

279 

280 Parameters 

281 ---------- 

282 string : `str` 

283 String containing content in YAML format 

284 

285 Returns 

286 ------- 

287 c : `Config` 

288 Newly-constructed Config. 

289 """ 

290 return cls.fromString(string, format="yaml") 

291 

292 def __initFromUri(self, path: Union[str, ButlerURI, Path]) -> None: 

293 """Load a file from a path or an URI. 

294 

295 Parameters 

296 ---------- 

297 path : `str` 

298 Path or a URI to a persisted config file. 

299 """ 

300 uri = ButlerURI(path) 

301 ext = uri.getExtension() 

302 if ext == ".yaml": 302 ↛ 309line 302 didn't jump to line 309, because the condition on line 302 was never false

303 log.debug("Opening YAML config file: %s", uri.geturl()) 

304 content = uri.read() 

305 # Use a stream so we can name it 

306 stream = io.BytesIO(content) 

307 stream.name = uri.geturl() 

308 self.__initFromYaml(stream) 

309 elif ext == ".json": 

310 log.debug("Opening JSON config file: %s", uri.geturl()) 

311 content = uri.read() 

312 self.__initFromJson(content) 

313 else: 

314 # This URI does not have a valid extension. It might be because 

315 # we ended up with a directory and not a file. Before we complain 

316 # about an extension, do an existence check. No need to do 

317 # the (possibly expensive) existence check in the default code 

318 # path above because we will find out soon enough that the file 

319 # is not there. 

320 if not uri.exists(): 

321 raise FileNotFoundError(f"Config location {uri} does not exist.") 

322 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

323 self.configFile = uri 

324 

325 def __initFromYaml(self, stream): 

326 """Loads a YAML config from any readable stream that contains one. 

327 

328 Parameters 

329 ---------- 

330 stream: `IO` or `str` 

331 Stream to pass to the YAML loader. Accepts anything that 

332 `yaml.load` accepts. This can include a string as well as an 

333 IO stream. 

334 

335 Raises 

336 ------ 

337 yaml.YAMLError 

338 If there is an error loading the file. 

339 """ 

340 content = yaml.load(stream, Loader=Loader) 

341 if content is None: 341 ↛ 342line 341 didn't jump to line 342, because the condition on line 341 was never true

342 content = {} 

343 self._data = content 

344 return self 

345 

346 def __initFromJson(self, stream): 

347 """Loads a JSON config from any readable stream that contains one. 

348 

349 Parameters 

350 ---------- 

351 stream: `IO` or `str` 

352 Stream to pass to the JSON loader. This can include a string as 

353 well as an IO stream. 

354 

355 Raises 

356 ------ 

357 TypeError: 

358 Raised if there is an error loading the content. 

359 """ 

360 if isinstance(stream, (bytes, str)): 

361 content = json.loads(stream) 

362 else: 

363 content = json.load(stream) 

364 if content is None: 

365 content = {} 

366 self._data = content 

367 return self 

368 

369 def _processExplicitIncludes(self): 

370 """Scan through the configuration searching for the special 

371 includeConfigs directive and process the includes.""" 

372 

373 # Search paths for config files 

374 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

375 if self.configFile is not None: 375 ↛ 383line 375 didn't jump to line 383, because the condition on line 375 was never false

376 if isinstance(self.configFile, ButlerURI): 376 ↛ 379line 376 didn't jump to line 379, because the condition on line 376 was never false

377 configDir = self.configFile.dirname() 

378 else: 

379 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

380 searchPaths.append(configDir) 

381 

382 # Ensure we know what delimiter to use 

383 names = self.nameTuples() 

384 for path in names: 

385 if path[-1] == self.includeKey: 385 ↛ 387line 385 didn't jump to line 387, because the condition on line 385 was never true

386 

387 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

388 basePath = path[:-1] 

389 

390 # Extract the includes and then delete them from the config 

391 includes = self[path] 

392 del self[path] 

393 

394 # Be consistent and convert to a list 

395 if not isinstance(includes, list): 

396 includes = [includes] 

397 

398 # Read each file assuming it is a reference to a file 

399 # The file can be relative to config file or cwd 

400 # ConfigSubset search paths are not used 

401 subConfigs = [] 

402 for fileName in includes: 

403 # Expand any shell variables -- this could be URI 

404 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

405 found = None 

406 if fileName.isabs(): 

407 found = fileName 

408 else: 

409 for dir in searchPaths: 

410 if isinstance(dir, ButlerURI): 

411 specific = dir.join(fileName.path) 

412 # Remote resource check might be expensive 

413 if specific.exists(): 

414 found = specific 

415 else: 

416 log.warning("Do not understand search path entry '%s' of type %s", 

417 dir, type(dir).__name__) 

418 if not found: 

419 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

420 

421 # Read the referenced Config as a Config 

422 subConfigs.append(type(self)(found)) 

423 

424 # Now we need to merge these sub configs with the current 

425 # information that was present in this node in the config 

426 # tree with precedence given to the explicit values 

427 newConfig = subConfigs.pop(0) 

428 for sc in subConfigs: 

429 newConfig.update(sc) 

430 

431 # Explicit values take precedence 

432 if not basePath: 

433 # This is an include at the root config 

434 newConfig.update(self) 

435 # Replace the current config 

436 self._data = newConfig._data 

437 else: 

438 newConfig.update(self[basePath]) 

439 # And reattach to the base config 

440 self[basePath] = newConfig 

441 

442 @staticmethod 

443 def _splitIntoKeys(key): 

444 r"""Split the argument for get/set/in into a hierarchical list. 

445 

446 Parameters 

447 ---------- 

448 key : `str` or iterable 

449 Argument given to get/set/in. If an iterable is provided it will 

450 be converted to a list. If the first character of the string 

451 is not an alphanumeric character then it will be used as the 

452 delimiter for the purposes of splitting the remainder of the 

453 string. If the delimiter is also in one of the keys then it 

454 can be escaped using ``\``. There is no default delimiter. 

455 

456 Returns 

457 ------- 

458 keys : `list` 

459 Hierarchical keys as a `list`. 

460 """ 

461 if isinstance(key, str): 

462 if not key[0].isalnum(): 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true

463 d = key[0] 

464 key = key[1:] 

465 else: 

466 return [key, ] 

467 escaped = f"\\{d}" 

468 temp = None 

469 if escaped in key: 

470 # Complain at the attempt to escape the escape 

471 doubled = fr"\{escaped}" 

472 if doubled in key: 

473 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

474 " is not yet supported.") 

475 # Replace with a character that won't be in the string 

476 temp = "\r" 

477 if temp in key or d == temp: 

478 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

479 " delimiter if escaping the delimiter") 

480 key = key.replace(escaped, temp) 

481 hierarchy = key.split(d) 

482 if temp: 

483 hierarchy = [h.replace(temp, d) for h in hierarchy] 

484 return hierarchy 

485 elif isinstance(key, collections.abc.Iterable): 485 ↛ 489line 485 didn't jump to line 489, because the condition on line 485 was never false

486 return list(key) 

487 else: 

488 # Not sure what this is so try it anyway 

489 return [key, ] 

490 

491 def _getKeyHierarchy(self, name): 

492 """Retrieve the key hierarchy for accessing the Config 

493 

494 Parameters 

495 ---------- 

496 name : `str` or `tuple` 

497 Delimited string or `tuple` of hierarchical keys. 

498 

499 Returns 

500 ------- 

501 hierarchy : `list` of `str` 

502 Hierarchy to use as a `list`. If the name is available directly 

503 as a key in the Config it will be used regardless of the presence 

504 of any nominal delimiter. 

505 """ 

506 if name in self._data: 

507 keys = [name, ] 

508 else: 

509 keys = self._splitIntoKeys(name) 

510 return keys 

511 

512 def _findInHierarchy(self, keys, create=False): 

513 """Look for hierarchy of keys in Config 

514 

515 Parameters 

516 ---------- 

517 keys : `list` or `tuple` 

518 Keys to search in hierarchy. 

519 create : `bool`, optional 

520 If `True`, if a part of the hierarchy does not exist, insert an 

521 empty `dict` into the hierarchy. 

522 

523 Returns 

524 ------- 

525 hierarchy : `list` 

526 List of the value corresponding to each key in the supplied 

527 hierarchy. Only keys that exist in the hierarchy will have 

528 a value. 

529 complete : `bool` 

530 `True` if the full hierarchy exists and the final element 

531 in ``hierarchy`` is the value of relevant value. 

532 """ 

533 d = self._data 

534 

535 def checkNextItem(k, d, create): 

536 """See if k is in d and if it is return the new child""" 

537 nextVal = None 

538 isThere = False 

539 if d is None: 539 ↛ 541line 539 didn't jump to line 541, because the condition on line 539 was never true

540 # We have gone past the end of the hierarchy 

541 pass 

542 elif isinstance(d, collections.abc.Sequence): 542 ↛ 547line 542 didn't jump to line 547, because the condition on line 542 was never true

543 # Check sequence first because for lists 

544 # __contains__ checks whether value is found in list 

545 # not whether the index exists in list. When we traverse 

546 # the hierarchy we are interested in the index. 

547 try: 

548 nextVal = d[int(k)] 

549 isThere = True 

550 except IndexError: 

551 pass 

552 except ValueError: 

553 isThere = k in d 

554 elif k in d: 

555 nextVal = d[k] 

556 isThere = True 

557 elif create: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true

558 d[k] = {} 

559 nextVal = d[k] 

560 isThere = True 

561 return nextVal, isThere 

562 

563 hierarchy = [] 

564 complete = True 

565 for k in keys: 

566 d, isThere = checkNextItem(k, d, create) 

567 if isThere: 

568 hierarchy.append(d) 

569 else: 

570 complete = False 

571 break 

572 

573 return hierarchy, complete 

574 

575 def __getitem__(self, name): 

576 # Override the split for the simple case where there is an exact 

577 # match. This allows `Config.items()` to work via a simple 

578 # __iter__ implementation that returns top level keys of 

579 # self._data. 

580 keys = self._getKeyHierarchy(name) 

581 

582 hierarchy, complete = self._findInHierarchy(keys) 

583 if not complete: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true

584 raise KeyError(f"{name} not found") 

585 data = hierarchy[-1] 

586 

587 if isinstance(data, collections.abc.Mapping): 

588 data = Config(data) 

589 # Ensure that child configs inherit the parent internal delimiter 

590 if self._D != Config._D: 590 ↛ 591line 590 didn't jump to line 591, because the condition on line 590 was never true

591 data._D = self._D 

592 return data 

593 

594 def __setitem__(self, name, value): 

595 keys = self._getKeyHierarchy(name) 

596 last = keys.pop() 

597 if isinstance(value, Config): 

598 value = copy.deepcopy(value._data) 

599 

600 hierarchy, complete = self._findInHierarchy(keys, create=True) 

601 if hierarchy: 

602 data = hierarchy[-1] 

603 else: 

604 data = self._data 

605 

606 try: 

607 data[last] = value 

608 except TypeError: 

609 data[int(last)] = value 

610 

611 def __contains__(self, key): 

612 keys = self._getKeyHierarchy(key) 

613 hierarchy, complete = self._findInHierarchy(keys) 

614 return complete 

615 

616 def __delitem__(self, key): 

617 keys = self._getKeyHierarchy(key) 

618 last = keys.pop() 

619 hierarchy, complete = self._findInHierarchy(keys) 

620 if complete: 620 ↛ 627line 620 didn't jump to line 627, because the condition on line 620 was never false

621 if hierarchy: 621 ↛ 622line 621 didn't jump to line 622, because the condition on line 621 was never true

622 data = hierarchy[-1] 

623 else: 

624 data = self._data 

625 del data[last] 

626 else: 

627 raise KeyError(f"{key} not found in Config") 

628 

629 def update(self, other): 

630 """Like dict.update, but will add or modify keys in nested dicts, 

631 instead of overwriting the nested dict entirely. 

632 

633 For example, for the given code: 

634 foo = {"a": {"b": 1}} 

635 foo.update({"a": {"c": 2}}) 

636 

637 Parameters 

638 ---------- 

639 other : `dict` or `Config` 

640 Source of configuration: 

641 

642 - If foo is a dict, then after the update foo == {"a": {"c": 2}} 

643 - But if foo is a Config, then after the update 

644 foo == {"a": {"b": 1, "c": 2}} 

645 """ 

646 def doUpdate(d, u): 

647 if not isinstance(u, collections.abc.Mapping) or \ 647 ↛ 649line 647 didn't jump to line 649, because the condition on line 647 was never true

648 not isinstance(d, collections.abc.MutableMapping): 

649 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

650 for k, v in u.items(): 

651 if isinstance(v, collections.abc.Mapping): 

652 d[k] = doUpdate(d.get(k, {}), v) 

653 else: 

654 d[k] = v 

655 return d 

656 doUpdate(self._data, other) 

657 

658 def merge(self, other): 

659 """Like Config.update, but will add keys & values from other that 

660 DO NOT EXIST in self. 

661 

662 Keys and values that already exist in self will NOT be overwritten. 

663 

664 Parameters 

665 ---------- 

666 other : `dict` or `Config` 

667 Source of configuration: 

668 """ 

669 if not isinstance(other, collections.abc.Mapping): 

670 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}") 

671 

672 # Convert the supplied mapping to a Config for consistency 

673 # This will do a deepcopy if it is already a Config 

674 otherCopy = Config(other) 

675 otherCopy.update(self) 

676 self._data = otherCopy._data 

677 

678 def nameTuples(self, topLevelOnly=False): 

679 """Get tuples representing the name hierarchies of all keys. 

680 

681 The tuples returned from this method are guaranteed to be usable 

682 to access items in the configuration object. 

683 

684 Parameters 

685 ---------- 

686 topLevelOnly : `bool`, optional 

687 If False, the default, a full hierarchy of names is returned. 

688 If True, only the top level are returned. 

689 

690 Returns 

691 ------- 

692 names : `list` of `tuple` of `str` 

693 List of all names present in the `Config` where each element 

694 in the list is a `tuple` of strings representing the hierarchy. 

695 """ 

696 if topLevelOnly: 696 ↛ 697line 696 didn't jump to line 697, because the condition on line 696 was never true

697 return list((k,) for k in self) 

698 

699 def getKeysAsTuples(d, keys, base): 

700 if isinstance(d, collections.abc.Sequence): 

701 theseKeys = range(len(d)) 

702 else: 

703 theseKeys = d.keys() 

704 for key in theseKeys: 

705 val = d[key] 

706 levelKey = base + (key,) if base is not None else (key,) 

707 keys.append(levelKey) 

708 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

709 and not isinstance(val, str): 

710 getKeysAsTuples(val, keys, levelKey) 

711 keys: List[Tuple[str, ...]] = [] 

712 getKeysAsTuples(self._data, keys, None) 

713 return keys 

714 

715 def names(self, topLevelOnly=False, delimiter=None): 

716 """Get a delimited name of all the keys in the hierarchy. 

717 

718 The values returned from this method are guaranteed to be usable 

719 to access items in the configuration object. 

720 

721 Parameters 

722 ---------- 

723 topLevelOnly : `bool`, optional 

724 If False, the default, a full hierarchy of names is returned. 

725 If True, only the top level are returned. 

726 delimiter : `str`, optional 

727 Delimiter to use when forming the keys. If the delimiter is 

728 present in any of the keys, it will be escaped in the returned 

729 names. If `None` given a delimiter will be automatically provided. 

730 The delimiter can not be alphanumeric. 

731 

732 Returns 

733 ------- 

734 names : `list` of `str` 

735 List of all names present in the `Config`. 

736 

737 Notes 

738 ----- 

739 This is different than the built-in method `dict.keys`, which will 

740 return only the first level keys. 

741 

742 Raises 

743 ------ 

744 ValueError: 

745 The supplied delimiter is alphanumeric. 

746 """ 

747 if topLevelOnly: 

748 return list(self.keys()) 

749 

750 # Get all the tuples of hierarchical keys 

751 nameTuples = self.nameTuples() 

752 

753 if delimiter is not None and delimiter.isalnum(): 

754 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

755 

756 if delimiter is None: 

757 # Start with something, and ensure it does not need to be 

758 # escaped (it is much easier to understand if not escaped) 

759 delimiter = self._D 

760 

761 # Form big string for easy check of delimiter clash 

762 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

763 

764 # Try a delimiter and keep trying until we get something that 

765 # works. 

766 ntries = 0 

767 while delimiter in combined: 

768 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.") 

769 ntries += 1 

770 

771 if ntries > 100: 

772 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

773 

774 # try another one 

775 while True: 

776 delimiter = chr(ord(delimiter)+1) 

777 if not delimiter.isalnum(): 

778 break 

779 

780 log.debug(f"Using delimiter {delimiter!r}") 

781 

782 # Form the keys, escaping the delimiter if necessary 

783 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

784 for k in nameTuples] 

785 return strings 

786 

787 def asArray(self, name): 

788 """Get a value as an array. 

789 

790 May contain one or more elements. 

791 

792 Parameters 

793 ---------- 

794 name : `str` 

795 Key to use to retrieve value. 

796 

797 Returns 

798 ------- 

799 array : `collections.abc.Sequence` 

800 The value corresponding to name, but guaranteed to be returned 

801 as a list with at least one element. If the value is a 

802 `~collections.abc.Sequence` (and not a `str`) the value itself 

803 will be returned, else the value will be the first element. 

804 """ 

805 val = self.get(name) 

806 if isinstance(val, str): 

807 val = [val] 

808 elif not isinstance(val, collections.abc.Sequence): 

809 val = [val] 

810 return val 

811 

812 def __eq__(self, other): 

813 if isinstance(other, Config): 

814 other = other._data 

815 return self._data == other 

816 

817 def __ne__(self, other): 

818 if isinstance(other, Config): 

819 other = other._data 

820 return self._data != other 

821 

822 ####### 

823 # i/o # 

824 

825 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

826 """Writes the config to an output stream. 

827 

828 Parameters 

829 ---------- 

830 output : `IO`, optional 

831 The stream to use for output. If `None` the serialized content 

832 will be returned. 

833 format : `str`, optional 

834 The format to use for the output. Can be "yaml" or "json". 

835 

836 Returns 

837 ------- 

838 serialized : `str` or `None` 

839 If a stream was given the stream will be used and the return 

840 value will be `None`. If the stream was `None` the 

841 serialization will be returned as a string. 

842 """ 

843 if format == "yaml": 

844 return yaml.safe_dump(self._data, output, default_flow_style=False) 

845 elif format == "json": 

846 if output is not None: 

847 json.dump(self._data, output, ensure_ascii=False) 

848 return None 

849 else: 

850 return json.dumps(self._data, ensure_ascii=False) 

851 raise ValueError(f"Unsupported format for Config serialization: {format}") 

852 

853 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

854 defaultFileName: str = "butler.yaml", 

855 overwrite: bool = True) -> None: 

856 """Writes the config to location pointed to by given URI. 

857 

858 Currently supports 's3' and 'file' URI schemes. 

859 

860 Parameters 

861 ---------- 

862 uri: `str` or `ButlerURI` 

863 URI of location where the Config will be written. 

864 updateFile : bool, optional 

865 If True and uri does not end on a filename with extension, will 

866 append `defaultFileName` to the target uri. True by default. 

867 defaultFileName : bool, optional 

868 The file name that will be appended to target uri if updateFile is 

869 True and uri does not end on a file with an extension. 

870 overwrite : bool, optional 

871 If True the configuration will be written even if it already 

872 exists at that location. 

873 """ 

874 # Make local copy of URI or create new one 

875 uri = ButlerURI(uri) 

876 

877 if updateFile and not uri.getExtension(): 

878 uri.updateFile(defaultFileName) 

879 

880 # Try to work out the format from the extension 

881 ext = uri.getExtension() 

882 format = ext[1:].lower() 

883 

884 output = self.dump(format=format) 

885 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

886 uri.write(output.encode(), overwrite=overwrite) 

887 self.configFile = uri 

888 

889 @staticmethod 

890 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True): 

891 """Generic helper function for updating specific config parameters. 

892 

893 Allows for named parameters to be set to new values in bulk, and 

894 for other values to be set by copying from a reference config. 

895 

896 Assumes that the supplied config is compatible with ``configType`` 

897 and will attach the updated values to the supplied config by 

898 looking for the related component key. It is assumed that 

899 ``config`` and ``full`` are from the same part of the 

900 configuration hierarchy. 

901 

902 Parameters 

903 ---------- 

904 configType : `ConfigSubset` 

905 Config type to use to extract relevant items from ``config``. 

906 config : `Config` 

907 A `Config` to update. Only the subset understood by 

908 the supplied `ConfigSubset` will be modified. Default values 

909 will not be inserted and the content will not be validated 

910 since mandatory keys are allowed to be missing until 

911 populated later by merging. 

912 full : `Config` 

913 A complete config with all defaults expanded that can be 

914 converted to a ``configType``. Read-only and will not be 

915 modified by this method. Values are read from here if 

916 ``toCopy`` is defined. 

917 

918 Repository-specific options that should not be obtained 

919 from defaults when Butler instances are constructed 

920 should be copied from ``full`` to ``config``. 

921 toUpdate : `dict`, optional 

922 A `dict` defining the keys to update and the new value to use. 

923 The keys and values can be any supported by `Config` 

924 assignment. 

925 toCopy : `tuple`, optional 

926 `tuple` of keys whose values should be copied from ``full`` 

927 into ``config``. 

928 overwrite : `bool`, optional 

929 If `False`, do not modify a value in ``config`` if the key 

930 already exists. Default is always to overwrite. 

931 

932 Raises 

933 ------ 

934 ValueError 

935 Neither ``toUpdate`` not ``toCopy`` were defined. 

936 """ 

937 if toUpdate is None and toCopy is None: 

938 raise ValueError("One of toUpdate or toCopy parameters must be set.") 

939 

940 # If this is a parent configuration then we need to ensure that 

941 # the supplied config has the relevant component key in it. 

942 # If this is a parent configuration we add in the stub entry 

943 # so that the ConfigSubset constructor will do the right thing. 

944 # We check full for this since that is guaranteed to be complete. 

945 if configType.component in full and configType.component not in config: 

946 config[configType.component] = {} 

947 

948 # Extract the part of the config we wish to update 

949 localConfig = configType(config, mergeDefaults=False, validate=False) 

950 

951 if toUpdate: 

952 for key, value in toUpdate.items(): 

953 if key in localConfig and not overwrite: 

954 log.debug("Not overriding key '%s' with value '%s' in config %s", 

955 key, value, localConfig.__class__.__name__) 

956 else: 

957 localConfig[key] = value 

958 

959 if toCopy: 

960 localFullConfig = configType(full, mergeDefaults=False) 

961 for key in toCopy: 

962 if key in localConfig and not overwrite: 

963 log.debug("Not overriding key '%s' from defaults in config %s", 

964 key, localConfig.__class__.__name__) 

965 else: 

966 localConfig[key] = localFullConfig[key] 

967 

968 # Reattach to parent if this is a child config 

969 if configType.component in config: 

970 config[configType.component] = localConfig 

971 else: 

972 config.update(localConfig) 

973 

974 def toDict(self): 

975 """Convert a `Config` to a standalone hierarchical `dict`. 

976 

977 Returns 

978 ------- 

979 d : `dict` 

980 The standalone hierarchical `dict` with any `Config` classes 

981 in the hierarchy converted to `dict`. 

982 

983 Notes 

984 ----- 

985 This can be useful when passing a Config to some code that 

986 expects native Python types. 

987 """ 

988 output = copy.deepcopy(self._data) 

989 for k, v in output.items(): 

990 if isinstance(v, Config): 

991 v = v.toDict() 

992 output[k] = v 

993 return output 

994 

995 

996class ConfigSubset(Config): 

997 """Config representing a subset of a more general configuration. 

998 

999 Subclasses define their own component and when given a configuration 

1000 that includes that component, the resulting configuration only includes 

1001 the subset. For example, your config might contain ``dimensions`` if it's 

1002 part of a global config and that subset will be stored. If ``dimensions`` 

1003 can not be found it is assumed that the entire contents of the 

1004 configuration should be used. 

1005 

1006 Default values are read from the environment or supplied search paths 

1007 using the default configuration file name specified in the subclass. 

1008 This allows a configuration class to be instantiated without any 

1009 additional arguments. 

1010 

1011 Additional validation can be specified to check for keys that are mandatory 

1012 in the configuration. 

1013 

1014 Parameters 

1015 ---------- 

1016 other : `Config` or `str` or `dict` 

1017 Argument specifying the configuration information as understood 

1018 by `Config` 

1019 validate : `bool`, optional 

1020 If `True` required keys will be checked to ensure configuration 

1021 consistency. 

1022 mergeDefaults : `bool`, optional 

1023 If `True` defaults will be read and the supplied config will 

1024 be combined with the defaults, with the supplied valiues taking 

1025 precedence. 

1026 searchPaths : `list` or `tuple`, optional 

1027 Explicit additional paths to search for defaults. They should 

1028 be supplied in priority order. These paths have higher priority 

1029 than those read from the environment in 

1030 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1031 the local file system or URIs, `ButlerURI`. 

1032 """ 

1033 

1034 component: ClassVar[Optional[str]] = None 

1035 """Component to use from supplied config. Can be None. If specified the 

1036 key is not required. Can be a full dot-separated path to a component. 

1037 """ 

1038 

1039 requiredKeys: ClassVar[Sequence[str]] = () 

1040 """Keys that are required to be specified in the configuration. 

1041 """ 

1042 

1043 defaultConfigFile: ClassVar[Optional[str]] = None 

1044 """Name of the file containing defaults for this config class. 

1045 """ 

1046 

1047 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1048 

1049 # Create a blank object to receive the defaults 

1050 # Once we have the defaults we then update with the external values 

1051 super().__init__() 

1052 

1053 # Create a standard Config rather than subset 

1054 externalConfig = Config(other) 

1055 

1056 # Select the part we need from it 

1057 # To simplify the use of !include we also check for the existence of 

1058 # component.component (since the included files can themselves 

1059 # include the component name) 

1060 if self.component is not None: 1060 ↛ 1069line 1060 didn't jump to line 1069, because the condition on line 1060 was never false

1061 doubled = (self.component, self.component) 

1062 # Must check for double depth first 

1063 if doubled in externalConfig: 1063 ↛ 1064line 1063 didn't jump to line 1064, because the condition on line 1063 was never true

1064 externalConfig = externalConfig[doubled] 

1065 elif self.component in externalConfig: 

1066 externalConfig._data = externalConfig._data[self.component] 

1067 

1068 # Default files read to create this configuration 

1069 self.filesRead = [] 

1070 

1071 # Assume we are not looking up child configurations 

1072 containerKey = None 

1073 

1074 # Sometimes we do not want to merge with defaults. 

1075 if mergeDefaults: 

1076 

1077 # Supplied search paths have highest priority 

1078 fullSearchPath = [] 

1079 if searchPaths: 1079 ↛ 1080line 1079 didn't jump to line 1080, because the condition on line 1079 was never true

1080 fullSearchPath.extend(searchPaths) 

1081 

1082 # Read default paths from enviroment 

1083 fullSearchPath.extend(self.defaultSearchPaths()) 

1084 

1085 # There are two places to find defaults for this particular config 

1086 # - The "defaultConfigFile" defined in the subclass 

1087 # - The class specified in the "cls" element in the config. 

1088 # Read cls after merging in case it changes. 

1089 if self.defaultConfigFile is not None: 1089 ↛ 1094line 1089 didn't jump to line 1094, because the condition on line 1089 was never false

1090 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1091 

1092 # Can have a class specification in the external config (priority) 

1093 # or from the defaults. 

1094 pytype = None 

1095 if "cls" in externalConfig: 1095 ↛ 1096line 1095 didn't jump to line 1096, because the condition on line 1095 was never true

1096 pytype = externalConfig["cls"] 

1097 elif "cls" in self: 1097 ↛ 1098line 1097 didn't jump to line 1098, because the condition on line 1097 was never true

1098 pytype = self["cls"] 

1099 

1100 if pytype is not None: 1100 ↛ 1101line 1100 didn't jump to line 1101, because the condition on line 1100 was never true

1101 try: 

1102 cls = doImport(pytype) 

1103 except ImportError as e: 

1104 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1105 defaultsFile = cls.defaultConfigFile 

1106 if defaultsFile is not None: 

1107 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1108 

1109 # Get the container key in case we need it 

1110 try: 

1111 containerKey = cls.containerKey 

1112 except AttributeError: 

1113 pass 

1114 

1115 # Now update this object with the external values so that the external 

1116 # values always override the defaults 

1117 self.update(externalConfig) 

1118 

1119 # If this configuration has child configurations of the same 

1120 # config class, we need to expand those defaults as well. 

1121 

1122 if mergeDefaults and containerKey is not None and containerKey in self: 1122 ↛ 1123line 1122 didn't jump to line 1123, because the condition on line 1122 was never true

1123 for idx, subConfig in enumerate(self[containerKey]): 

1124 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1125 mergeDefaults=mergeDefaults, 

1126 searchPaths=searchPaths) 

1127 

1128 if validate: 

1129 self.validate() 

1130 

1131 @classmethod 

1132 def defaultSearchPaths(cls): 

1133 """Read the environment to determine search paths to use for global 

1134 defaults. 

1135 

1136 Global defaults, at lowest priority, are found in the ``config`` 

1137 directory of the butler source tree. Additional defaults can be 

1138 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1139 which is a PATH-like variable where paths at the front of the list 

1140 have priority over those later. 

1141 

1142 Returns 

1143 ------- 

1144 paths : `list` 

1145 Returns a list of paths to search. The returned order is in 

1146 priority with the highest priority paths first. The butler config 

1147 configuration resources will not be included here but will 

1148 always be searched last. 

1149 

1150 Notes 

1151 ----- 

1152 The environment variable is split on the standard ``:`` path separator. 

1153 This currently makes it incompatible with usage of URIs. 

1154 """ 

1155 # We can pick up defaults from multiple search paths 

1156 # We fill defaults by using the butler config path and then 

1157 # the config path environment variable in reverse order. 

1158 defaultsPaths: List[Union[str, ButlerURI]] = [] 

1159 

1160 if CONFIG_PATH in os.environ: 1160 ↛ 1161line 1160 didn't jump to line 1161, because the condition on line 1160 was never true

1161 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1162 defaultsPaths.extend(externalPaths) 

1163 

1164 # Add the package defaults as a resource 

1165 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1166 forceDirectory=True)) 

1167 return defaultsPaths 

1168 

1169 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1170 """Search the supplied paths, merging the configuration values 

1171 

1172 The values read will override values currently stored in the object. 

1173 Every file found in the path will be read, such that the earlier 

1174 path entries have higher priority. 

1175 

1176 Parameters 

1177 ---------- 

1178 searchPaths : `list` of `ButlerURI`, `str` 

1179 Paths to search for the supplied configFile. This path 

1180 is the priority order, such that files read from the 

1181 first path entry will be selected over those read from 

1182 a later path. Can contain `str` referring to the local file 

1183 system or a URI string. 

1184 configFile : `ButlerURI` 

1185 File to locate in path. If absolute path it will be read 

1186 directly and the search path will not be used. Can be a URI 

1187 to an explicit resource (which will ignore the search path) 

1188 which is assumed to exist. 

1189 """ 

1190 uri = ButlerURI(configFile) 

1191 if uri.isabs() and uri.exists(): 1191 ↛ 1193line 1191 didn't jump to line 1193, because the condition on line 1191 was never true

1192 # Assume this resource exists 

1193 self._updateWithOtherConfigFile(configFile) 

1194 self.filesRead.append(configFile) 

1195 else: 

1196 # Reverse order so that high priority entries 

1197 # update the object last. 

1198 for pathDir in reversed(searchPaths): 

1199 if isinstance(pathDir, (str, ButlerURI)): 1199 ↛ 1206line 1199 didn't jump to line 1206, because the condition on line 1199 was never false

1200 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1201 file = pathDir.join(configFile) 

1202 if file.exists(): 1202 ↛ 1198line 1202 didn't jump to line 1198, because the condition on line 1202 was never false

1203 self.filesRead.append(file) 

1204 self._updateWithOtherConfigFile(file) 

1205 else: 

1206 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1207 

1208 def _updateWithOtherConfigFile(self, file): 

1209 """Read in some defaults and update. 

1210 

1211 Update the configuration by reading the supplied file as a config 

1212 of this class, and merging such that these values override the 

1213 current values. Contents of the external config are not validated. 

1214 

1215 Parameters 

1216 ---------- 

1217 file : `Config`, `str`, `ButlerURI`, or `dict` 

1218 Entity that can be converted to a `ConfigSubset`. 

1219 """ 

1220 # Use this class to read the defaults so that subsetting can happen 

1221 # correctly. 

1222 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1223 self.update(externalConfig) 

1224 

1225 def validate(self): 

1226 """Check that mandatory keys are present in this configuration. 

1227 

1228 Ignored if ``requiredKeys`` is empty.""" 

1229 # Validation 

1230 missing = [k for k in self.requiredKeys if k not in self._data] 

1231 if missing: 1231 ↛ 1232line 1231 didn't jump to line 1232, because the condition on line 1231 was never true

1232 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")