Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from yaml.representer import Representer 

37import io 

38from typing import Any, Dict, List, Sequence, Optional, ClassVar, IO, Tuple, Union 

39 

40from lsst.utils import doImport 

41from ._butlerUri import ButlerURI 

42 

43yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

44 

45 

46# Config module logger 

47log = logging.getLogger(__name__) 

48 

49# PATH-like environment variable to use for defaults. 

50CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

51 

52try: 

53 yamlLoader = yaml.CSafeLoader 

54except AttributeError: 

55 # Not all installations have the C library 

56 # (but assume for mypy's sake that they're the same) 

57 yamlLoader = yaml.SafeLoader # type: ignore 

58 

59 

60class Loader(yamlLoader): 

61 """YAML Loader that supports file include directives 

62 

63 Uses ``!include`` directive in a YAML file to point to another 

64 YAML file to be included. The path in the include directive is relative 

65 to the file containing that directive. 

66 

67 storageClasses: !include storageClasses.yaml 

68 

69 Examples 

70 -------- 

71 >>> with open("document.yaml", "r") as f: 

72 data = yaml.load(f, Loader=Loader) 

73 

74 Notes 

75 ----- 

76 See https://davidchall.github.io/yaml-includes.html 

77 """ 

78 

79 def __init__(self, stream): 

80 super().__init__(stream) 

81 # if this is a string and not a stream we may well lack a name 

82 try: 

83 self._root = ButlerURI(stream.name) 

84 except AttributeError: 

85 # No choice but to assume a local filesystem 

86 self._root = ButlerURI("no-file.yaml") 

87 Loader.add_constructor("!include", Loader.include) 

88 

89 def include(self, node): 

90 result: Union[List[Any], Dict[str, Any]] 

91 if isinstance(node, yaml.ScalarNode): 

92 return self.extractFile(self.construct_scalar(node)) 

93 

94 elif isinstance(node, yaml.SequenceNode): 

95 result = [] 

96 for filename in self.construct_sequence(node): 

97 result.append(self.extractFile(filename)) 

98 return result 

99 

100 elif isinstance(node, yaml.MappingNode): 

101 result = {} 

102 for k, v in self.construct_mapping(node).items(): 

103 result[k] = self.extractFile(v) 

104 return result 

105 

106 else: 

107 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

108 raise yaml.constructor.ConstructorError 

109 

110 def extractFile(self, filename): 

111 # It is possible for the !include to point to an explicit URI 

112 # instead of a relative URI, therefore we first see if it is 

113 # scheme-less or not. If it has a scheme we use it directly 

114 # if it is scheme-less we use it relative to the file root. 

115 requesteduri = ButlerURI(filename, forceAbsolute=False) 

116 

117 if requesteduri.scheme: 

118 fileuri = requesteduri 

119 else: 

120 fileuri = copy.copy(self._root) 

121 fileuri.updateFile(filename) 

122 

123 log.debug("Opening YAML file via !include: %s", fileuri) 

124 

125 # Read all the data from the resource 

126 data = fileuri.read() 

127 

128 # Store the bytes into a BytesIO so we can attach a .name 

129 stream = io.BytesIO(data) 

130 stream.name = fileuri.geturl() 

131 return yaml.load(stream, Loader) 

132 

133 

134class Config(collections.abc.MutableMapping): 

135 r"""Implements a datatype that is used by `Butler` for configuration 

136 parameters. 

137 

138 It is essentially a `dict` with key/value pairs, including nested dicts 

139 (as values). In fact, it can be initialized with a `dict`. 

140 This is explained next: 

141 

142 Config extends the `dict` api so that hierarchical values may be accessed 

143 with delimited notation or as a tuple. If a string is given the delimiter 

144 is picked up from the first character in that string. For example, 

145 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

146 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

147 If the first character is alphanumeric, no delimiter will be used. 

148 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

149 Unicode characters can be used as the delimiter for distinctiveness if 

150 required. 

151 

152 If a key in the hierarchy starts with a non-alphanumeric character care 

153 should be used to ensure that either the tuple interface is used or 

154 a distinct delimiter is always given in string form. 

155 

156 Finally, the delimiter can be escaped if it is part of a key and also 

157 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

158 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

159 always better to use a different delimiter in these cases. 

160 

161 Note that adding a multi-level key implicitly creates any nesting levels 

162 that do not exist, but removing multi-level keys does not automatically 

163 remove empty nesting levels. As a result: 

164 

165 >>> c = Config() 

166 >>> c[".a.b"] = 1 

167 >>> del c[".a.b"] 

168 >>> c["a"] 

169 Config({'a': {}}) 

170 

171 Storage formats supported: 

172 

173 - yaml: read and write is supported. 

174 - json: read and write is supported but no ``!include`` directive. 

175 

176 Parameters 

177 ---------- 

178 other : `str` or `Config` or `dict` or `ButlerURI` 

179 Other source of configuration, can be: 

180 

181 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

182 with ".yaml". 

183 - (`Config`) Copies the other Config's values into this one. 

184 - (`dict`) Copies the values from the dict into this Config. 

185 

186 If `None` is provided an empty `Config` will be created. 

187 """ 

188 

189 _D: str = "→" 

190 """Default internal delimiter to use for components in the hierarchy when 

191 constructing keys for external use (see `Config.names()`).""" 

192 

193 includeKey: ClassVar[str] = "includeConfigs" 

194 """Key used to indicate that another config should be included at this 

195 part of the hierarchy.""" 

196 

197 resourcesPackage: str = "lsst.daf.butler" 

198 """Package to search for default configuration data. The resources 

199 themselves will be within a ``configs`` resource hierarchy.""" 

200 

201 def __init__(self, other=None): 

202 self._data: Dict[str, Any] = {} 

203 self.configFile = None 

204 

205 if other is None: 

206 return 

207 

208 if isinstance(other, Config): 

209 self._data = copy.deepcopy(other._data) 

210 self.configFile = other.configFile 

211 elif isinstance(other, collections.abc.Mapping): 

212 self.update(other) 

213 elif isinstance(other, (str, ButlerURI)): 213 ↛ 220line 213 didn't jump to line 220, because the condition on line 213 was never false

214 # if other is a string, assume it is a file path/URI 

215 self.__initFromUri(other) 

216 self._processExplicitIncludes() 

217 else: 

218 # if the config specified by other could not be recognized raise 

219 # a runtime error. 

220 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

221 

222 def ppprint(self): 

223 """helper function for debugging, prints a config out in a readable 

224 way in the debugger. 

225 

226 use: pdb> print(myConfigObject.ppprint()) 

227 

228 Returns 

229 ------- 

230 s : `str` 

231 A prettyprint formatted string representing the config 

232 """ 

233 return pprint.pformat(self._data, indent=2, width=1) 

234 

235 def __repr__(self): 

236 return f"{type(self).__name__}({self._data!r})" 

237 

238 def __str__(self): 

239 return self.ppprint() 

240 

241 def __len__(self): 

242 return len(self._data) 

243 

244 def __iter__(self): 

245 return iter(self._data) 

246 

247 def copy(self): 

248 return type(self)(self) 

249 

250 @classmethod 

251 def fromString(cls, string: str, format: str = "yaml") -> Config: 

252 """Create a new Config instance from a serialized string. 

253 

254 Parameters 

255 ---------- 

256 string : `str` 

257 String containing content in specified format 

258 format : `str`, optional 

259 Format of the supplied string. Can be ``json`` or ``yaml``. 

260 

261 Returns 

262 ------- 

263 c : `Config` 

264 Newly-constructed Config. 

265 """ 

266 if format == "yaml": 

267 new_config = cls().__initFromYaml(string) 

268 elif format == "json": 

269 new_config = cls().__initFromJson(string) 

270 else: 

271 raise ValueError(f"Unexpected format of string: {format}") 

272 new_config._processExplicitIncludes() 

273 return new_config 

274 

275 @classmethod 

276 def fromYaml(cls, string: str) -> Config: 

277 """Create a new Config instance from a YAML string. 

278 

279 Parameters 

280 ---------- 

281 string : `str` 

282 String containing content in YAML format 

283 

284 Returns 

285 ------- 

286 c : `Config` 

287 Newly-constructed Config. 

288 """ 

289 return cls.fromString(string, format="yaml") 

290 

291 def __initFromUri(self, path: Union[str, ButlerURI]) -> None: 

292 """Load a file from a path or an URI. 

293 

294 Parameters 

295 ---------- 

296 path : `str` 

297 Path or a URI to a persisted config file. 

298 """ 

299 uri = ButlerURI(path) 

300 ext = uri.getExtension() 

301 if ext == ".yaml": 301 ↛ 308line 301 didn't jump to line 308, because the condition on line 301 was never false

302 log.debug("Opening YAML config file: %s", uri.geturl()) 

303 content = uri.read() 

304 # Use a stream so we can name it 

305 stream = io.BytesIO(content) 

306 stream.name = uri.geturl() 

307 self.__initFromYaml(stream) 

308 elif ext == ".json": 

309 log.debug("Opening JSON config file: %s", uri.geturl()) 

310 content = uri.read() 

311 self.__initFromJson(content) 

312 else: 

313 # This URI does not have a valid extension. It might be because 

314 # we ended up with a directory and not a file. Before we complain 

315 # about an extension, do an existence check. No need to do 

316 # the (possibly expensive) existence check in the default code 

317 # path above because we will find out soon enough that the file 

318 # is not there. 

319 if not uri.exists(): 

320 raise FileNotFoundError(f"Config location {uri} does not exist.") 

321 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

322 self.configFile = uri 

323 

324 def __initFromYaml(self, stream): 

325 """Loads a YAML config from any readable stream that contains one. 

326 

327 Parameters 

328 ---------- 

329 stream: `IO` or `str` 

330 Stream to pass to the YAML loader. Accepts anything that 

331 `yaml.load` accepts. This can include a string as well as an 

332 IO stream. 

333 

334 Raises 

335 ------ 

336 yaml.YAMLError 

337 If there is an error loading the file. 

338 """ 

339 content = yaml.load(stream, Loader=Loader) 

340 if content is None: 340 ↛ 341line 340 didn't jump to line 341, because the condition on line 340 was never true

341 content = {} 

342 self._data = content 

343 return self 

344 

345 def __initFromJson(self, stream): 

346 """Loads a JSON config from any readable stream that contains one. 

347 

348 Parameters 

349 ---------- 

350 stream: `IO` or `str` 

351 Stream to pass to the JSON loader. This can include a string as 

352 well as an IO stream. 

353 

354 Raises 

355 ------ 

356 TypeError: 

357 Raised if there is an error loading the content. 

358 """ 

359 if isinstance(stream, (bytes, str)): 

360 content = json.loads(stream) 

361 else: 

362 content = json.load(stream) 

363 if content is None: 

364 content = {} 

365 self._data = content 

366 return self 

367 

368 def _processExplicitIncludes(self): 

369 """Scan through the configuration searching for the special 

370 includeConfigs directive and process the includes.""" 

371 

372 # Search paths for config files 

373 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

374 if self.configFile is not None: 374 ↛ 382line 374 didn't jump to line 382, because the condition on line 374 was never false

375 if isinstance(self.configFile, ButlerURI): 375 ↛ 378line 375 didn't jump to line 378, because the condition on line 375 was never false

376 configDir = self.configFile.dirname() 

377 else: 

378 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

379 searchPaths.append(configDir) 

380 

381 # Ensure we know what delimiter to use 

382 names = self.nameTuples() 

383 for path in names: 

384 if path[-1] == self.includeKey: 384 ↛ 386line 384 didn't jump to line 386, because the condition on line 384 was never true

385 

386 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

387 basePath = path[:-1] 

388 

389 # Extract the includes and then delete them from the config 

390 includes = self[path] 

391 del self[path] 

392 

393 # Be consistent and convert to a list 

394 if not isinstance(includes, list): 

395 includes = [includes] 

396 

397 # Read each file assuming it is a reference to a file 

398 # The file can be relative to config file or cwd 

399 # ConfigSubset search paths are not used 

400 subConfigs = [] 

401 for fileName in includes: 

402 # Expand any shell variables -- this could be URI 

403 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

404 found = None 

405 if fileName.isabs(): 

406 found = fileName 

407 else: 

408 for dir in searchPaths: 

409 if isinstance(dir, ButlerURI): 

410 specific = dir.join(fileName.path) 

411 # Remote resource check might be expensive 

412 if specific.exists(): 

413 found = specific 

414 else: 

415 log.warning("Do not understand search path entry '%s' of type %s", 

416 dir, type(dir).__name__) 

417 if not found: 

418 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

419 

420 # Read the referenced Config as a Config 

421 subConfigs.append(type(self)(found)) 

422 

423 # Now we need to merge these sub configs with the current 

424 # information that was present in this node in the config 

425 # tree with precedence given to the explicit values 

426 newConfig = subConfigs.pop(0) 

427 for sc in subConfigs: 

428 newConfig.update(sc) 

429 

430 # Explicit values take precedence 

431 if not basePath: 

432 # This is an include at the root config 

433 newConfig.update(self) 

434 # Replace the current config 

435 self._data = newConfig._data 

436 else: 

437 newConfig.update(self[basePath]) 

438 # And reattach to the base config 

439 self[basePath] = newConfig 

440 

441 @staticmethod 

442 def _splitIntoKeys(key): 

443 r"""Split the argument for get/set/in into a hierarchical list. 

444 

445 Parameters 

446 ---------- 

447 key : `str` or iterable 

448 Argument given to get/set/in. If an iterable is provided it will 

449 be converted to a list. If the first character of the string 

450 is not an alphanumeric character then it will be used as the 

451 delimiter for the purposes of splitting the remainder of the 

452 string. If the delimiter is also in one of the keys then it 

453 can be escaped using ``\``. There is no default delimiter. 

454 

455 Returns 

456 ------- 

457 keys : `list` 

458 Hierarchical keys as a `list`. 

459 """ 

460 if isinstance(key, str): 

461 if not key[0].isalnum(): 461 ↛ 462line 461 didn't jump to line 462, because the condition on line 461 was never true

462 d = key[0] 

463 key = key[1:] 

464 else: 

465 return [key, ] 

466 escaped = f"\\{d}" 

467 temp = None 

468 if escaped in key: 

469 # Complain at the attempt to escape the escape 

470 doubled = fr"\{escaped}" 

471 if doubled in key: 

472 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

473 " is not yet supported.") 

474 # Replace with a character that won't be in the string 

475 temp = "\r" 

476 if temp in key or d == temp: 

477 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

478 " delimiter if escaping the delimiter") 

479 key = key.replace(escaped, temp) 

480 hierarchy = key.split(d) 

481 if temp: 

482 hierarchy = [h.replace(temp, d) for h in hierarchy] 

483 return hierarchy 

484 elif isinstance(key, collections.abc.Iterable): 484 ↛ 488line 484 didn't jump to line 488, because the condition on line 484 was never false

485 return list(key) 

486 else: 

487 # Not sure what this is so try it anyway 

488 return [key, ] 

489 

490 def _getKeyHierarchy(self, name): 

491 """Retrieve the key hierarchy for accessing the Config 

492 

493 Parameters 

494 ---------- 

495 name : `str` or `tuple` 

496 Delimited string or `tuple` of hierarchical keys. 

497 

498 Returns 

499 ------- 

500 hierarchy : `list` of `str` 

501 Hierarchy to use as a `list`. If the name is available directly 

502 as a key in the Config it will be used regardless of the presence 

503 of any nominal delimiter. 

504 """ 

505 if name in self._data: 

506 keys = [name, ] 

507 else: 

508 keys = self._splitIntoKeys(name) 

509 return keys 

510 

511 def _findInHierarchy(self, keys, create=False): 

512 """Look for hierarchy of keys in Config 

513 

514 Parameters 

515 ---------- 

516 keys : `list` or `tuple` 

517 Keys to search in hierarchy. 

518 create : `bool`, optional 

519 If `True`, if a part of the hierarchy does not exist, insert an 

520 empty `dict` into the hierarchy. 

521 

522 Returns 

523 ------- 

524 hierarchy : `list` 

525 List of the value corresponding to each key in the supplied 

526 hierarchy. Only keys that exist in the hierarchy will have 

527 a value. 

528 complete : `bool` 

529 `True` if the full hierarchy exists and the final element 

530 in ``hierarchy`` is the value of relevant value. 

531 """ 

532 d = self._data 

533 

534 def checkNextItem(k, d, create): 

535 """See if k is in d and if it is return the new child""" 

536 nextVal = None 

537 isThere = False 

538 if d is None: 538 ↛ 540line 538 didn't jump to line 540, because the condition on line 538 was never true

539 # We have gone past the end of the hierarchy 

540 pass 

541 elif isinstance(d, collections.abc.Sequence): 541 ↛ 546line 541 didn't jump to line 546, because the condition on line 541 was never true

542 # Check sequence first because for lists 

543 # __contains__ checks whether value is found in list 

544 # not whether the index exists in list. When we traverse 

545 # the hierarchy we are interested in the index. 

546 try: 

547 nextVal = d[int(k)] 

548 isThere = True 

549 except IndexError: 

550 pass 

551 except ValueError: 

552 isThere = k in d 

553 elif k in d: 

554 nextVal = d[k] 

555 isThere = True 

556 elif create: 556 ↛ 557line 556 didn't jump to line 557, because the condition on line 556 was never true

557 d[k] = {} 

558 nextVal = d[k] 

559 isThere = True 

560 return nextVal, isThere 

561 

562 hierarchy = [] 

563 complete = True 

564 for k in keys: 

565 d, isThere = checkNextItem(k, d, create) 

566 if isThere: 

567 hierarchy.append(d) 

568 else: 

569 complete = False 

570 break 

571 

572 return hierarchy, complete 

573 

574 def __getitem__(self, name): 

575 # Override the split for the simple case where there is an exact 

576 # match. This allows `Config.items()` to work via a simple 

577 # __iter__ implementation that returns top level keys of 

578 # self._data. 

579 keys = self._getKeyHierarchy(name) 

580 

581 hierarchy, complete = self._findInHierarchy(keys) 

582 if not complete: 582 ↛ 583line 582 didn't jump to line 583, because the condition on line 582 was never true

583 raise KeyError(f"{name} not found") 

584 data = hierarchy[-1] 

585 

586 if isinstance(data, collections.abc.Mapping): 

587 data = Config(data) 

588 # Ensure that child configs inherit the parent internal delimiter 

589 if self._D != Config._D: 589 ↛ 590line 589 didn't jump to line 590, because the condition on line 589 was never true

590 data._D = self._D 

591 return data 

592 

593 def __setitem__(self, name, value): 

594 keys = self._getKeyHierarchy(name) 

595 last = keys.pop() 

596 if isinstance(value, Config): 

597 value = copy.deepcopy(value._data) 

598 

599 hierarchy, complete = self._findInHierarchy(keys, create=True) 

600 if hierarchy: 

601 data = hierarchy[-1] 

602 else: 

603 data = self._data 

604 

605 try: 

606 data[last] = value 

607 except TypeError: 

608 data[int(last)] = value 

609 

610 def __contains__(self, key): 

611 keys = self._getKeyHierarchy(key) 

612 hierarchy, complete = self._findInHierarchy(keys) 

613 return complete 

614 

615 def __delitem__(self, key): 

616 keys = self._getKeyHierarchy(key) 

617 last = keys.pop() 

618 hierarchy, complete = self._findInHierarchy(keys) 

619 if complete: 619 ↛ 626line 619 didn't jump to line 626, because the condition on line 619 was never false

620 if hierarchy: 620 ↛ 621line 620 didn't jump to line 621, because the condition on line 620 was never true

621 data = hierarchy[-1] 

622 else: 

623 data = self._data 

624 del data[last] 

625 else: 

626 raise KeyError(f"{key} not found in Config") 

627 

628 def update(self, other): 

629 """Like dict.update, but will add or modify keys in nested dicts, 

630 instead of overwriting the nested dict entirely. 

631 

632 For example, for the given code: 

633 foo = {"a": {"b": 1}} 

634 foo.update({"a": {"c": 2}}) 

635 

636 Parameters 

637 ---------- 

638 other : `dict` or `Config` 

639 Source of configuration: 

640 

641 - If foo is a dict, then after the update foo == {"a": {"c": 2}} 

642 - But if foo is a Config, then after the update 

643 foo == {"a": {"b": 1, "c": 2}} 

644 """ 

645 def doUpdate(d, u): 

646 if not isinstance(u, collections.abc.Mapping) or \ 646 ↛ 648line 646 didn't jump to line 648, because the condition on line 646 was never true

647 not isinstance(d, collections.abc.MutableMapping): 

648 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

649 for k, v in u.items(): 

650 if isinstance(v, collections.abc.Mapping): 

651 d[k] = doUpdate(d.get(k, {}), v) 

652 else: 

653 d[k] = v 

654 return d 

655 doUpdate(self._data, other) 

656 

657 def merge(self, other): 

658 """Like Config.update, but will add keys & values from other that 

659 DO NOT EXIST in self. 

660 

661 Keys and values that already exist in self will NOT be overwritten. 

662 

663 Parameters 

664 ---------- 

665 other : `dict` or `Config` 

666 Source of configuration: 

667 """ 

668 otherCopy = copy.deepcopy(other) 

669 otherCopy.update(self) 

670 self._data = otherCopy._data 

671 

672 def nameTuples(self, topLevelOnly=False): 

673 """Get tuples representing the name hierarchies of all keys. 

674 

675 The tuples returned from this method are guaranteed to be usable 

676 to access items in the configuration object. 

677 

678 Parameters 

679 ---------- 

680 topLevelOnly : `bool`, optional 

681 If False, the default, a full hierarchy of names is returned. 

682 If True, only the top level are returned. 

683 

684 Returns 

685 ------- 

686 names : `list` of `tuple` of `str` 

687 List of all names present in the `Config` where each element 

688 in the list is a `tuple` of strings representing the hierarchy. 

689 """ 

690 if topLevelOnly: 690 ↛ 691line 690 didn't jump to line 691, because the condition on line 690 was never true

691 return list((k,) for k in self) 

692 

693 def getKeysAsTuples(d, keys, base): 

694 if isinstance(d, collections.abc.Sequence): 

695 theseKeys = range(len(d)) 

696 else: 

697 theseKeys = d.keys() 

698 for key in theseKeys: 

699 val = d[key] 

700 levelKey = base + (key,) if base is not None else (key,) 

701 keys.append(levelKey) 

702 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

703 and not isinstance(val, str): 

704 getKeysAsTuples(val, keys, levelKey) 

705 keys: List[Tuple[str, ...]] = [] 

706 getKeysAsTuples(self._data, keys, None) 

707 return keys 

708 

709 def names(self, topLevelOnly=False, delimiter=None): 

710 """Get a delimited name of all the keys in the hierarchy. 

711 

712 The values returned from this method are guaranteed to be usable 

713 to access items in the configuration object. 

714 

715 Parameters 

716 ---------- 

717 topLevelOnly : `bool`, optional 

718 If False, the default, a full hierarchy of names is returned. 

719 If True, only the top level are returned. 

720 delimiter : `str`, optional 

721 Delimiter to use when forming the keys. If the delimiter is 

722 present in any of the keys, it will be escaped in the returned 

723 names. If `None` given a delimiter will be automatically provided. 

724 The delimiter can not be alphanumeric. 

725 

726 Returns 

727 ------- 

728 names : `list` of `str` 

729 List of all names present in the `Config`. 

730 

731 Notes 

732 ----- 

733 This is different than the built-in method `dict.keys`, which will 

734 return only the first level keys. 

735 

736 Raises 

737 ------ 

738 ValueError: 

739 The supplied delimiter is alphanumeric. 

740 """ 

741 if topLevelOnly: 

742 return list(self.keys()) 

743 

744 # Get all the tuples of hierarchical keys 

745 nameTuples = self.nameTuples() 

746 

747 if delimiter is not None and delimiter.isalnum(): 

748 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

749 

750 if delimiter is None: 

751 # Start with something, and ensure it does not need to be 

752 # escaped (it is much easier to understand if not escaped) 

753 delimiter = self._D 

754 

755 # Form big string for easy check of delimiter clash 

756 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

757 

758 # Try a delimiter and keep trying until we get something that 

759 # works. 

760 ntries = 0 

761 while delimiter in combined: 

762 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.") 

763 ntries += 1 

764 

765 if ntries > 100: 

766 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

767 

768 # try another one 

769 while True: 

770 delimiter = chr(ord(delimiter)+1) 

771 if not delimiter.isalnum(): 

772 break 

773 

774 log.debug(f"Using delimiter {delimiter!r}") 

775 

776 # Form the keys, escaping the delimiter if necessary 

777 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

778 for k in nameTuples] 

779 return strings 

780 

781 def asArray(self, name): 

782 """Get a value as an array. 

783 

784 May contain one or more elements. 

785 

786 Parameters 

787 ---------- 

788 name : `str` 

789 Key to use to retrieve value. 

790 

791 Returns 

792 ------- 

793 array : `collections.abc.Sequence` 

794 The value corresponding to name, but guaranteed to be returned 

795 as a list with at least one element. If the value is a 

796 `~collections.abc.Sequence` (and not a `str`) the value itself 

797 will be returned, else the value will be the first element. 

798 """ 

799 val = self.get(name) 

800 if isinstance(val, str): 

801 val = [val] 

802 elif not isinstance(val, collections.abc.Sequence): 

803 val = [val] 

804 return val 

805 

806 def __eq__(self, other): 

807 if isinstance(other, Config): 

808 other = other._data 

809 return self._data == other 

810 

811 def __ne__(self, other): 

812 if isinstance(other, Config): 

813 other = other._data 

814 return self._data != other 

815 

816 ####### 

817 # i/o # 

818 

819 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

820 """Writes the config to an output stream. 

821 

822 Parameters 

823 ---------- 

824 output : `IO`, optional 

825 The stream to use for output. If `None` the serialized content 

826 will be returned. 

827 format : `str`, optional 

828 The format to use for the output. Can be "yaml" or "json". 

829 

830 Returns 

831 ------- 

832 serialized : `str` or `None` 

833 If a stream was given the stream will be used and the return 

834 value will be `None`. If the stream was `None` the 

835 serialization will be returned as a string. 

836 """ 

837 if format == "yaml": 

838 return yaml.safe_dump(self._data, output, default_flow_style=False) 

839 elif format == "json": 

840 if output is not None: 

841 json.dump(self._data, output, ensure_ascii=False) 

842 return None 

843 else: 

844 return json.dumps(self._data, ensure_ascii=False) 

845 raise ValueError(f"Unsupported format for Config serialization: {format}") 

846 

847 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

848 defaultFileName: str = "butler.yaml", 

849 overwrite: bool = True) -> None: 

850 """Writes the config to location pointed to by given URI. 

851 

852 Currently supports 's3' and 'file' URI schemes. 

853 

854 Parameters 

855 ---------- 

856 uri: `str` or `ButlerURI` 

857 URI of location where the Config will be written. 

858 updateFile : bool, optional 

859 If True and uri does not end on a filename with extension, will 

860 append `defaultFileName` to the target uri. True by default. 

861 defaultFileName : bool, optional 

862 The file name that will be appended to target uri if updateFile is 

863 True and uri does not end on a file with an extension. 

864 overwrite : bool, optional 

865 If True the configuration will be written even if it already 

866 exists at that location. 

867 """ 

868 # Make local copy of URI or create new one 

869 uri = ButlerURI(uri) 

870 

871 if updateFile and not uri.getExtension(): 

872 uri.updateFile(defaultFileName) 

873 

874 # Try to work out the format from the extension 

875 ext = uri.getExtension() 

876 format = ext[1:].lower() 

877 

878 output = self.dump(format=format) 

879 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

880 uri.write(output.encode(), overwrite=overwrite) 

881 self.configFile = uri 

882 

883 @staticmethod 

884 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True): 

885 """Generic helper function for updating specific config parameters. 

886 

887 Allows for named parameters to be set to new values in bulk, and 

888 for other values to be set by copying from a reference config. 

889 

890 Assumes that the supplied config is compatible with ``configType`` 

891 and will attach the updated values to the supplied config by 

892 looking for the related component key. It is assumed that 

893 ``config`` and ``full`` are from the same part of the 

894 configuration hierarchy. 

895 

896 Parameters 

897 ---------- 

898 configType : `ConfigSubset` 

899 Config type to use to extract relevant items from ``config``. 

900 config : `Config` 

901 A `Config` to update. Only the subset understood by 

902 the supplied `ConfigSubset` will be modified. Default values 

903 will not be inserted and the content will not be validated 

904 since mandatory keys are allowed to be missing until 

905 populated later by merging. 

906 full : `Config` 

907 A complete config with all defaults expanded that can be 

908 converted to a ``configType``. Read-only and will not be 

909 modified by this method. Values are read from here if 

910 ``toCopy`` is defined. 

911 

912 Repository-specific options that should not be obtained 

913 from defaults when Butler instances are constructed 

914 should be copied from ``full`` to ``config``. 

915 toUpdate : `dict`, optional 

916 A `dict` defining the keys to update and the new value to use. 

917 The keys and values can be any supported by `Config` 

918 assignment. 

919 toCopy : `tuple`, optional 

920 `tuple` of keys whose values should be copied from ``full`` 

921 into ``config``. 

922 overwrite : `bool`, optional 

923 If `False`, do not modify a value in ``config`` if the key 

924 already exists. Default is always to overwrite. 

925 

926 Raises 

927 ------ 

928 ValueError 

929 Neither ``toUpdate`` not ``toCopy`` were defined. 

930 """ 

931 if toUpdate is None and toCopy is None: 

932 raise ValueError("One of toUpdate or toCopy parameters must be set.") 

933 

934 # If this is a parent configuration then we need to ensure that 

935 # the supplied config has the relevant component key in it. 

936 # If this is a parent configuration we add in the stub entry 

937 # so that the ConfigSubset constructor will do the right thing. 

938 # We check full for this since that is guaranteed to be complete. 

939 if configType.component in full and configType.component not in config: 

940 config[configType.component] = {} 

941 

942 # Extract the part of the config we wish to update 

943 localConfig = configType(config, mergeDefaults=False, validate=False) 

944 

945 if toUpdate: 

946 for key, value in toUpdate.items(): 

947 if key in localConfig and not overwrite: 

948 log.debug("Not overriding key '%s' with value '%s' in config %s", 

949 key, value, localConfig.__class__.__name__) 

950 else: 

951 localConfig[key] = value 

952 

953 if toCopy: 

954 localFullConfig = configType(full, mergeDefaults=False) 

955 for key in toCopy: 

956 if key in localConfig and not overwrite: 

957 log.debug("Not overriding key '%s' from defaults in config %s", 

958 key, localConfig.__class__.__name__) 

959 else: 

960 localConfig[key] = localFullConfig[key] 

961 

962 # Reattach to parent if this is a child config 

963 if configType.component in config: 

964 config[configType.component] = localConfig 

965 else: 

966 config.update(localConfig) 

967 

968 def toDict(self): 

969 """Convert a `Config` to a standalone hierarchical `dict`. 

970 

971 Returns 

972 ------- 

973 d : `dict` 

974 The standalone hierarchical `dict` with any `Config` classes 

975 in the hierarchy converted to `dict`. 

976 

977 Notes 

978 ----- 

979 This can be useful when passing a Config to some code that 

980 expects native Python types. 

981 """ 

982 output = copy.deepcopy(self._data) 

983 for k, v in output.items(): 

984 if isinstance(v, Config): 

985 v = v.toDict() 

986 output[k] = v 

987 return output 

988 

989 

990class ConfigSubset(Config): 

991 """Config representing a subset of a more general configuration. 

992 

993 Subclasses define their own component and when given a configuration 

994 that includes that component, the resulting configuration only includes 

995 the subset. For example, your config might contain ``dimensions`` if it's 

996 part of a global config and that subset will be stored. If ``dimensions`` 

997 can not be found it is assumed that the entire contents of the 

998 configuration should be used. 

999 

1000 Default values are read from the environment or supplied search paths 

1001 using the default configuration file name specified in the subclass. 

1002 This allows a configuration class to be instantiated without any 

1003 additional arguments. 

1004 

1005 Additional validation can be specified to check for keys that are mandatory 

1006 in the configuration. 

1007 

1008 Parameters 

1009 ---------- 

1010 other : `Config` or `str` or `dict` 

1011 Argument specifying the configuration information as understood 

1012 by `Config` 

1013 validate : `bool`, optional 

1014 If `True` required keys will be checked to ensure configuration 

1015 consistency. 

1016 mergeDefaults : `bool`, optional 

1017 If `True` defaults will be read and the supplied config will 

1018 be combined with the defaults, with the supplied valiues taking 

1019 precedence. 

1020 searchPaths : `list` or `tuple`, optional 

1021 Explicit additional paths to search for defaults. They should 

1022 be supplied in priority order. These paths have higher priority 

1023 than those read from the environment in 

1024 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1025 the local file system or URIs, `ButlerURI`. 

1026 """ 

1027 

1028 component: ClassVar[Optional[str]] = None 

1029 """Component to use from supplied config. Can be None. If specified the 

1030 key is not required. Can be a full dot-separated path to a component. 

1031 """ 

1032 

1033 requiredKeys: ClassVar[Sequence[str]] = () 

1034 """Keys that are required to be specified in the configuration. 

1035 """ 

1036 

1037 defaultConfigFile: ClassVar[Optional[str]] = None 

1038 """Name of the file containing defaults for this config class. 

1039 """ 

1040 

1041 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1042 

1043 # Create a blank object to receive the defaults 

1044 # Once we have the defaults we then update with the external values 

1045 super().__init__() 

1046 

1047 # Create a standard Config rather than subset 

1048 externalConfig = Config(other) 

1049 

1050 # Select the part we need from it 

1051 # To simplify the use of !include we also check for the existence of 

1052 # component.component (since the included files can themselves 

1053 # include the component name) 

1054 if self.component is not None: 1054 ↛ 1063line 1054 didn't jump to line 1063, because the condition on line 1054 was never false

1055 doubled = (self.component, self.component) 

1056 # Must check for double depth first 

1057 if doubled in externalConfig: 1057 ↛ 1058line 1057 didn't jump to line 1058, because the condition on line 1057 was never true

1058 externalConfig = externalConfig[doubled] 

1059 elif self.component in externalConfig: 

1060 externalConfig._data = externalConfig._data[self.component] 

1061 

1062 # Default files read to create this configuration 

1063 self.filesRead = [] 

1064 

1065 # Assume we are not looking up child configurations 

1066 containerKey = None 

1067 

1068 # Sometimes we do not want to merge with defaults. 

1069 if mergeDefaults: 

1070 

1071 # Supplied search paths have highest priority 

1072 fullSearchPath = [] 

1073 if searchPaths: 1073 ↛ 1074line 1073 didn't jump to line 1074, because the condition on line 1073 was never true

1074 fullSearchPath.extend(searchPaths) 

1075 

1076 # Read default paths from enviroment 

1077 fullSearchPath.extend(self.defaultSearchPaths()) 

1078 

1079 # There are two places to find defaults for this particular config 

1080 # - The "defaultConfigFile" defined in the subclass 

1081 # - The class specified in the "cls" element in the config. 

1082 # Read cls after merging in case it changes. 

1083 if self.defaultConfigFile is not None: 1083 ↛ 1088line 1083 didn't jump to line 1088, because the condition on line 1083 was never false

1084 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1085 

1086 # Can have a class specification in the external config (priority) 

1087 # or from the defaults. 

1088 pytype = None 

1089 if "cls" in externalConfig: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true

1090 pytype = externalConfig["cls"] 

1091 elif "cls" in self: 1091 ↛ 1092line 1091 didn't jump to line 1092, because the condition on line 1091 was never true

1092 pytype = self["cls"] 

1093 

1094 if pytype is not None: 1094 ↛ 1095line 1094 didn't jump to line 1095, because the condition on line 1094 was never true

1095 try: 

1096 cls = doImport(pytype) 

1097 except ImportError as e: 

1098 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1099 defaultsFile = cls.defaultConfigFile 

1100 if defaultsFile is not None: 

1101 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1102 

1103 # Get the container key in case we need it 

1104 try: 

1105 containerKey = cls.containerKey 

1106 except AttributeError: 

1107 pass 

1108 

1109 # Now update this object with the external values so that the external 

1110 # values always override the defaults 

1111 self.update(externalConfig) 

1112 

1113 # If this configuration has child configurations of the same 

1114 # config class, we need to expand those defaults as well. 

1115 

1116 if mergeDefaults and containerKey is not None and containerKey in self: 1116 ↛ 1117line 1116 didn't jump to line 1117, because the condition on line 1116 was never true

1117 for idx, subConfig in enumerate(self[containerKey]): 

1118 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1119 mergeDefaults=mergeDefaults, 

1120 searchPaths=searchPaths) 

1121 

1122 if validate: 

1123 self.validate() 

1124 

1125 @classmethod 

1126 def defaultSearchPaths(cls): 

1127 """Read the environment to determine search paths to use for global 

1128 defaults. 

1129 

1130 Global defaults, at lowest priority, are found in the ``config`` 

1131 directory of the butler source tree. Additional defaults can be 

1132 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1133 which is a PATH-like variable where paths at the front of the list 

1134 have priority over those later. 

1135 

1136 Returns 

1137 ------- 

1138 paths : `list` 

1139 Returns a list of paths to search. The returned order is in 

1140 priority with the highest priority paths first. The butler config 

1141 configuration resources will not be included here but will 

1142 always be searched last. 

1143 

1144 Notes 

1145 ----- 

1146 The environment variable is split on the standard ``:`` path separator. 

1147 This currently makes it incompatible with usage of URIs. 

1148 """ 

1149 # We can pick up defaults from multiple search paths 

1150 # We fill defaults by using the butler config path and then 

1151 # the config path environment variable in reverse order. 

1152 defaultsPaths: List[Union[str, ButlerURI]] = [] 

1153 

1154 if CONFIG_PATH in os.environ: 1154 ↛ 1155line 1154 didn't jump to line 1155, because the condition on line 1154 was never true

1155 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1156 defaultsPaths.extend(externalPaths) 

1157 

1158 # Add the package defaults as a resource 

1159 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1160 forceDirectory=True)) 

1161 return defaultsPaths 

1162 

1163 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1164 """Search the supplied paths, merging the configuration values 

1165 

1166 The values read will override values currently stored in the object. 

1167 Every file found in the path will be read, such that the earlier 

1168 path entries have higher priority. 

1169 

1170 Parameters 

1171 ---------- 

1172 searchPaths : `list` of `ButlerURI`, `str` 

1173 Paths to search for the supplied configFile. This path 

1174 is the priority order, such that files read from the 

1175 first path entry will be selected over those read from 

1176 a later path. Can contain `str` referring to the local file 

1177 system or a URI string. 

1178 configFile : `ButlerURI` 

1179 File to locate in path. If absolute path it will be read 

1180 directly and the search path will not be used. Can be a URI 

1181 to an explicit resource (which will ignore the search path) 

1182 which is assumed to exist. 

1183 """ 

1184 uri = ButlerURI(configFile) 

1185 if uri.isabs() and uri.exists(): 1185 ↛ 1187line 1185 didn't jump to line 1187, because the condition on line 1185 was never true

1186 # Assume this resource exists 

1187 self._updateWithOtherConfigFile(configFile) 

1188 self.filesRead.append(configFile) 

1189 else: 

1190 # Reverse order so that high priority entries 

1191 # update the object last. 

1192 for pathDir in reversed(searchPaths): 

1193 if isinstance(pathDir, (str, ButlerURI)): 1193 ↛ 1200line 1193 didn't jump to line 1200, because the condition on line 1193 was never false

1194 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1195 file = pathDir.join(configFile) 

1196 if file.exists(): 1196 ↛ 1192line 1196 didn't jump to line 1192, because the condition on line 1196 was never false

1197 self.filesRead.append(file) 

1198 self._updateWithOtherConfigFile(file) 

1199 else: 

1200 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1201 

1202 def _updateWithOtherConfigFile(self, file): 

1203 """Read in some defaults and update. 

1204 

1205 Update the configuration by reading the supplied file as a config 

1206 of this class, and merging such that these values override the 

1207 current values. Contents of the external config are not validated. 

1208 

1209 Parameters 

1210 ---------- 

1211 file : `Config`, `str`, `ButlerURI`, or `dict` 

1212 Entity that can be converted to a `ConfigSubset`. 

1213 """ 

1214 # Use this class to read the defaults so that subsetting can happen 

1215 # correctly. 

1216 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1217 self.update(externalConfig) 

1218 

1219 def validate(self): 

1220 """Check that mandatory keys are present in this configuration. 

1221 

1222 Ignored if ``requiredKeys`` is empty.""" 

1223 # Validation 

1224 missing = [k for k in self.requiredKeys if k not in self._data] 

1225 if missing: 1225 ↛ 1226line 1225 didn't jump to line 1226, because the condition on line 1225 was never true

1226 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")