Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from yaml.representer import Representer 

37import io 

38from typing import Sequence, Optional, ClassVar, IO, Union 

39 

40from lsst.utils import doImport 

41from ._butlerUri import ButlerURI 

42 

43yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

44 

45 

46# Config module logger 

47log = logging.getLogger(__name__) 

48 

49# PATH-like environment variable to use for defaults. 

50CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

51 

52try: 

53 yamlLoader = yaml.CSafeLoader 

54except AttributeError: 

55 # Not all installations have the C library 

56 yamlLoader = yaml.SafeLoader 

57 

58 

59class Loader(yamlLoader): 

60 """YAML Loader that supports file include directives 

61 

62 Uses ``!include`` directive in a YAML file to point to another 

63 YAML file to be included. The path in the include directive is relative 

64 to the file containing that directive. 

65 

66 storageClasses: !include storageClasses.yaml 

67 

68 Examples 

69 -------- 

70 >>> with open("document.yaml", "r") as f: 

71 data = yaml.load(f, Loader=Loader) 

72 

73 Notes 

74 ----- 

75 See https://davidchall.github.io/yaml-includes.html 

76 """ 

77 

78 def __init__(self, stream): 

79 super().__init__(stream) 

80 # if this is a string and not a stream we may well lack a name 

81 try: 

82 self._root = ButlerURI(stream.name) 

83 except AttributeError: 

84 # No choice but to assume a local filesystem 

85 self._root = ButlerURI("no-file.yaml") 

86 Loader.add_constructor("!include", Loader.include) 

87 

88 def include(self, node): 

89 if isinstance(node, yaml.ScalarNode): 

90 return self.extractFile(self.construct_scalar(node)) 

91 

92 elif isinstance(node, yaml.SequenceNode): 

93 result = [] 

94 for filename in self.construct_sequence(node): 

95 result.append(self.extractFile(filename)) 

96 return result 

97 

98 elif isinstance(node, yaml.MappingNode): 

99 result = {} 

100 for k, v in self.construct_mapping(node).items(): 

101 result[k] = self.extractFile(v) 

102 return result 

103 

104 else: 

105 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

106 raise yaml.constructor.ConstructorError 

107 

108 def extractFile(self, filename): 

109 # It is possible for the !include to point to an explicit URI 

110 # instead of a relative URI, therefore we first see if it is 

111 # scheme-less or not. If it has a scheme we use it directly 

112 # if it is scheme-less we use it relative to the file root. 

113 requesteduri = ButlerURI(filename, forceAbsolute=False) 

114 

115 if requesteduri.scheme: 

116 fileuri = requesteduri 

117 else: 

118 fileuri = copy.copy(self._root) 

119 fileuri.updateFile(filename) 

120 

121 log.debug("Opening YAML file via !include: %s", fileuri) 

122 

123 # Read all the data from the resource 

124 data = fileuri.read() 

125 

126 # Store the bytes into a BytesIO so we can attach a .name 

127 stream = io.BytesIO(data) 

128 stream.name = fileuri.geturl() 

129 return yaml.load(stream, Loader) 

130 

131 

132class Config(collections.abc.MutableMapping): 

133 r"""Implements a datatype that is used by `Butler` for configuration 

134 parameters. 

135 

136 It is essentially a `dict` with key/value pairs, including nested dicts 

137 (as values). In fact, it can be initialized with a `dict`. 

138 This is explained next: 

139 

140 Config extends the `dict` api so that hierarchical values may be accessed 

141 with delimited notation or as a tuple. If a string is given the delimiter 

142 is picked up from the first character in that string. For example, 

143 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

144 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

145 If the first character is alphanumeric, no delimiter will be used. 

146 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

147 Unicode characters can be used as the delimiter for distinctiveness if 

148 required. 

149 

150 If a key in the hierarchy starts with a non-alphanumeric character care 

151 should be used to ensure that either the tuple interface is used or 

152 a distinct delimiter is always given in string form. 

153 

154 Finally, the delimiter can be escaped if it is part of a key and also 

155 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

156 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

157 always better to use a different delimiter in these cases. 

158 

159 Note that adding a multi-level key implicitly creates any nesting levels 

160 that do not exist, but removing multi-level keys does not automatically 

161 remove empty nesting levels. As a result: 

162 

163 >>> c = Config() 

164 >>> c[".a.b"] = 1 

165 >>> del c[".a.b"] 

166 >>> c["a"] 

167 Config({'a': {}}) 

168 

169 Storage formats supported: 

170 

171 - yaml: read and write is supported. 

172 - json: read and write is supported but no ``!include`` directive. 

173 

174 Parameters 

175 ---------- 

176 other : `str` or `Config` or `dict` or `ButlerURI` 

177 Other source of configuration, can be: 

178 

179 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

180 with ".yaml". 

181 - (`Config`) Copies the other Config's values into this one. 

182 - (`dict`) Copies the values from the dict into this Config. 

183 

184 If `None` is provided an empty `Config` will be created. 

185 """ 

186 

187 _D: ClassVar[str] = "→" 

188 """Default internal delimiter to use for components in the hierarchy when 

189 constructing keys for external use (see `Config.names()`).""" 

190 

191 includeKey: ClassVar[str] = "includeConfigs" 

192 """Key used to indicate that another config should be included at this 

193 part of the hierarchy.""" 

194 

195 resourcesPackage: str = "lsst.daf.butler" 

196 """Package to search for default configuration data. The resources 

197 themselves will be within a ``configs`` resource hierarchy.""" 

198 

199 def __init__(self, other=None): 

200 self._data = {} 

201 self.configFile = None 

202 

203 if other is None: 

204 return 

205 

206 if isinstance(other, Config): 

207 self._data = copy.deepcopy(other._data) 

208 self.configFile = other.configFile 

209 elif isinstance(other, collections.abc.Mapping): 

210 self.update(other) 

211 elif isinstance(other, (str, ButlerURI)): 211 ↛ 218line 211 didn't jump to line 218, because the condition on line 211 was never false

212 # if other is a string, assume it is a file path/URI 

213 self.__initFromUri(other) 

214 self._processExplicitIncludes() 

215 else: 

216 # if the config specified by other could not be recognized raise 

217 # a runtime error. 

218 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

219 

220 def ppprint(self): 

221 """helper function for debugging, prints a config out in a readable 

222 way in the debugger. 

223 

224 use: pdb> print(myConfigObject.ppprint()) 

225 

226 Returns 

227 ------- 

228 s : `str` 

229 A prettyprint formatted string representing the config 

230 """ 

231 return pprint.pformat(self._data, indent=2, width=1) 

232 

233 def __repr__(self): 

234 return f"{type(self).__name__}({self._data!r})" 

235 

236 def __str__(self): 

237 return self.ppprint() 

238 

239 def __len__(self): 

240 return len(self._data) 

241 

242 def __iter__(self): 

243 return iter(self._data) 

244 

245 def copy(self): 

246 return type(self)(self) 

247 

248 @classmethod 

249 def fromString(cls, string: str, format: str = "yaml") -> Config: 

250 """Create a new Config instance from a serialized string. 

251 

252 Parameters 

253 ---------- 

254 string : `str` 

255 String containing content in specified format 

256 format : `str`, optional 

257 Format of the supplied string. Can be ``json`` or ``yaml``. 

258 

259 Returns 

260 ------- 

261 c : `Config` 

262 Newly-constructed Config. 

263 """ 

264 if format == "yaml": 

265 new_config = cls().__initFromYaml(string) 

266 elif format == "json": 

267 new_config = cls().__initFromJson(string) 

268 else: 

269 raise ValueError(f"Unexpected format of string: {format}") 

270 new_config._processExplicitIncludes() 

271 return new_config 

272 

273 @classmethod 

274 def fromYaml(cls, string: str) -> Config: 

275 """Create a new Config instance from a YAML string. 

276 

277 Parameters 

278 ---------- 

279 string : `str` 

280 String containing content in YAML format 

281 

282 Returns 

283 ------- 

284 c : `Config` 

285 Newly-constructed Config. 

286 """ 

287 return cls.fromString(string, format="yaml") 

288 

289 def __initFromUri(self, path: str) -> None: 

290 """Load a file from a path or an URI. 

291 

292 Parameters 

293 ---------- 

294 path : `str` 

295 Path or a URI to a persisted config file. 

296 """ 

297 uri = ButlerURI(path) 

298 ext = uri.getExtension() 

299 if ext == ".yaml": 299 ↛ 306line 299 didn't jump to line 306, because the condition on line 299 was never false

300 log.debug("Opening YAML config file: %s", uri.geturl()) 

301 content = uri.read() 

302 # Use a stream so we can name it 

303 stream = io.BytesIO(content) 

304 stream.name = uri.geturl() 

305 self.__initFromYaml(stream) 

306 elif ext == ".json": 

307 log.debug("Opening JSON config file: %s", uri.geturl()) 

308 content = uri.read() 

309 self.__initFromJson(content) 

310 else: 

311 # This URI does not have a valid extension. It might be because 

312 # we ended up with a directory and not a file. Before we complain 

313 # about an extension, do an existence check. No need to do 

314 # the (possibly expensive) existence check in the default code 

315 # path above because we will find out soon enough that the file 

316 # is not there. 

317 if not uri.exists(): 

318 raise FileNotFoundError(f"Config location {uri} does not exist.") 

319 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

320 self.configFile = uri 

321 

322 def __initFromYaml(self, stream): 

323 """Loads a YAML config from any readable stream that contains one. 

324 

325 Parameters 

326 ---------- 

327 stream: `IO` or `str` 

328 Stream to pass to the YAML loader. Accepts anything that 

329 `yaml.load` accepts. This can include a string as well as an 

330 IO stream. 

331 

332 Raises 

333 ------ 

334 yaml.YAMLError 

335 If there is an error loading the file. 

336 """ 

337 content = yaml.load(stream, Loader=Loader) 

338 if content is None: 338 ↛ 339line 338 didn't jump to line 339, because the condition on line 338 was never true

339 content = {} 

340 self._data = content 

341 return self 

342 

343 def __initFromJson(self, stream): 

344 """Loads a JSON config from any readable stream that contains one. 

345 

346 Parameters 

347 ---------- 

348 stream: `IO` or `str` 

349 Stream to pass to the JSON loader. This can include a string as 

350 well as an IO stream. 

351 

352 Raises 

353 ------ 

354 TypeError: 

355 Raised if there is an error loading the content. 

356 """ 

357 if isinstance(stream, (bytes, str)): 

358 content = json.loads(stream) 

359 else: 

360 content = json.load(stream) 

361 if content is None: 

362 content = {} 

363 self._data = content 

364 return self 

365 

366 def _processExplicitIncludes(self): 

367 """Scan through the configuration searching for the special 

368 includeConfigs directive and process the includes.""" 

369 

370 # Search paths for config files 

371 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

372 if self.configFile is not None: 372 ↛ 380line 372 didn't jump to line 380, because the condition on line 372 was never false

373 if isinstance(self.configFile, ButlerURI): 373 ↛ 376line 373 didn't jump to line 376, because the condition on line 373 was never false

374 configDir = self.configFile.dirname() 

375 else: 

376 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

377 searchPaths.append(configDir) 

378 

379 # Ensure we know what delimiter to use 

380 names = self.nameTuples() 

381 for path in names: 

382 if path[-1] == self.includeKey: 382 ↛ 384line 382 didn't jump to line 384, because the condition on line 382 was never true

383 

384 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

385 basePath = path[:-1] 

386 

387 # Extract the includes and then delete them from the config 

388 includes = self[path] 

389 del self[path] 

390 

391 # Be consistent and convert to a list 

392 if not isinstance(includes, list): 

393 includes = [includes] 

394 

395 # Read each file assuming it is a reference to a file 

396 # The file can be relative to config file or cwd 

397 # ConfigSubset search paths are not used 

398 subConfigs = [] 

399 for fileName in includes: 

400 # Expand any shell variables -- this could be URI 

401 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

402 found = None 

403 if fileName.isabs(): 

404 found = fileName 

405 else: 

406 for dir in searchPaths: 

407 if isinstance(dir, ButlerURI): 

408 specific = dir.join(fileName.path) 

409 # Remote resource check might be expensive 

410 if specific.exists(): 

411 found = specific 

412 else: 

413 log.warning("Do not understand search path entry '%s' of type %s", 

414 dir, type(dir).__name__) 

415 if not found: 

416 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

417 

418 # Read the referenced Config as a Config 

419 subConfigs.append(type(self)(found)) 

420 

421 # Now we need to merge these sub configs with the current 

422 # information that was present in this node in the config 

423 # tree with precedence given to the explicit values 

424 newConfig = subConfigs.pop(0) 

425 for sc in subConfigs: 

426 newConfig.update(sc) 

427 

428 # Explicit values take precedence 

429 if not basePath: 

430 # This is an include at the root config 

431 newConfig.update(self) 

432 # Replace the current config 

433 self._data = newConfig._data 

434 else: 

435 newConfig.update(self[basePath]) 

436 # And reattach to the base config 

437 self[basePath] = newConfig 

438 

439 @staticmethod 

440 def _splitIntoKeys(key): 

441 r"""Split the argument for get/set/in into a hierarchical list. 

442 

443 Parameters 

444 ---------- 

445 key : `str` or iterable 

446 Argument given to get/set/in. If an iterable is provided it will 

447 be converted to a list. If the first character of the string 

448 is not an alphanumeric character then it will be used as the 

449 delimiter for the purposes of splitting the remainder of the 

450 string. If the delimiter is also in one of the keys then it 

451 can be escaped using ``\``. There is no default delimiter. 

452 

453 Returns 

454 ------- 

455 keys : `list` 

456 Hierarchical keys as a `list`. 

457 """ 

458 if isinstance(key, str): 

459 if not key[0].isalnum(): 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true

460 d = key[0] 

461 key = key[1:] 

462 else: 

463 return [key, ] 

464 escaped = f"\\{d}" 

465 temp = None 

466 if escaped in key: 

467 # Complain at the attempt to escape the escape 

468 doubled = fr"\{escaped}" 

469 if doubled in key: 

470 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

471 " is not yet supported.") 

472 # Replace with a character that won't be in the string 

473 temp = "\r" 

474 if temp in key or d == temp: 

475 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

476 " delimiter if escaping the delimiter") 

477 key = key.replace(escaped, temp) 

478 hierarchy = key.split(d) 

479 if temp: 

480 hierarchy = [h.replace(temp, d) for h in hierarchy] 

481 return hierarchy 

482 elif isinstance(key, collections.abc.Iterable): 482 ↛ 486line 482 didn't jump to line 486, because the condition on line 482 was never false

483 return list(key) 

484 else: 

485 # Not sure what this is so try it anyway 

486 return [key, ] 

487 

488 def _getKeyHierarchy(self, name): 

489 """Retrieve the key hierarchy for accessing the Config 

490 

491 Parameters 

492 ---------- 

493 name : `str` or `tuple` 

494 Delimited string or `tuple` of hierarchical keys. 

495 

496 Returns 

497 ------- 

498 hierarchy : `list` of `str` 

499 Hierarchy to use as a `list`. If the name is available directly 

500 as a key in the Config it will be used regardless of the presence 

501 of any nominal delimiter. 

502 """ 

503 if name in self._data: 

504 keys = [name, ] 

505 else: 

506 keys = self._splitIntoKeys(name) 

507 return keys 

508 

509 def _findInHierarchy(self, keys, create=False): 

510 """Look for hierarchy of keys in Config 

511 

512 Parameters 

513 ---------- 

514 keys : `list` or `tuple` 

515 Keys to search in hierarchy. 

516 create : `bool`, optional 

517 If `True`, if a part of the hierarchy does not exist, insert an 

518 empty `dict` into the hierarchy. 

519 

520 Returns 

521 ------- 

522 hierarchy : `list` 

523 List of the value corresponding to each key in the supplied 

524 hierarchy. Only keys that exist in the hierarchy will have 

525 a value. 

526 complete : `bool` 

527 `True` if the full hierarchy exists and the final element 

528 in ``hierarchy`` is the value of relevant value. 

529 """ 

530 d = self._data 

531 

532 def checkNextItem(k, d, create): 

533 """See if k is in d and if it is return the new child""" 

534 nextVal = None 

535 isThere = False 

536 if d is None: 536 ↛ 538line 536 didn't jump to line 538, because the condition on line 536 was never true

537 # We have gone past the end of the hierarchy 

538 pass 

539 elif isinstance(d, collections.abc.Sequence): 539 ↛ 544line 539 didn't jump to line 544, because the condition on line 539 was never true

540 # Check sequence first because for lists 

541 # __contains__ checks whether value is found in list 

542 # not whether the index exists in list. When we traverse 

543 # the hierarchy we are interested in the index. 

544 try: 

545 nextVal = d[int(k)] 

546 isThere = True 

547 except IndexError: 

548 pass 

549 except ValueError: 

550 isThere = k in d 

551 elif k in d: 

552 nextVal = d[k] 

553 isThere = True 

554 elif create: 554 ↛ 555line 554 didn't jump to line 555, because the condition on line 554 was never true

555 d[k] = {} 

556 nextVal = d[k] 

557 isThere = True 

558 return nextVal, isThere 

559 

560 hierarchy = [] 

561 complete = True 

562 for k in keys: 

563 d, isThere = checkNextItem(k, d, create) 

564 if isThere: 

565 hierarchy.append(d) 

566 else: 

567 complete = False 

568 break 

569 

570 return hierarchy, complete 

571 

572 def __getitem__(self, name): 

573 # Override the split for the simple case where there is an exact 

574 # match. This allows `Config.items()` to work via a simple 

575 # __iter__ implementation that returns top level keys of 

576 # self._data. 

577 keys = self._getKeyHierarchy(name) 

578 

579 hierarchy, complete = self._findInHierarchy(keys) 

580 if not complete: 580 ↛ 581line 580 didn't jump to line 581, because the condition on line 580 was never true

581 raise KeyError(f"{name} not found") 

582 data = hierarchy[-1] 

583 

584 if isinstance(data, collections.abc.Mapping): 

585 data = Config(data) 

586 # Ensure that child configs inherit the parent internal delimiter 

587 if self._D != Config._D: 587 ↛ 588line 587 didn't jump to line 588, because the condition on line 587 was never true

588 data._D = self._D 

589 return data 

590 

591 def __setitem__(self, name, value): 

592 keys = self._getKeyHierarchy(name) 

593 last = keys.pop() 

594 if isinstance(value, Config): 

595 value = copy.deepcopy(value._data) 

596 

597 hierarchy, complete = self._findInHierarchy(keys, create=True) 

598 if hierarchy: 

599 data = hierarchy[-1] 

600 else: 

601 data = self._data 

602 

603 try: 

604 data[last] = value 

605 except TypeError: 

606 data[int(last)] = value 

607 

608 def __contains__(self, key): 

609 keys = self._getKeyHierarchy(key) 

610 hierarchy, complete = self._findInHierarchy(keys) 

611 return complete 

612 

613 def __delitem__(self, key): 

614 keys = self._getKeyHierarchy(key) 

615 last = keys.pop() 

616 hierarchy, complete = self._findInHierarchy(keys) 

617 if complete: 617 ↛ 624line 617 didn't jump to line 624, because the condition on line 617 was never false

618 if hierarchy: 618 ↛ 619line 618 didn't jump to line 619, because the condition on line 618 was never true

619 data = hierarchy[-1] 

620 else: 

621 data = self._data 

622 del data[last] 

623 else: 

624 raise KeyError(f"{key} not found in Config") 

625 

626 def update(self, other): 

627 """Like dict.update, but will add or modify keys in nested dicts, 

628 instead of overwriting the nested dict entirely. 

629 

630 For example, for the given code: 

631 foo = {"a": {"b": 1}} 

632 foo.update({"a": {"c": 2}}) 

633 

634 Parameters 

635 ---------- 

636 other : `dict` or `Config` 

637 Source of configuration: 

638 

639 - If foo is a dict, then after the update foo == {"a": {"c": 2}} 

640 - But if foo is a Config, then after the update 

641 foo == {"a": {"b": 1, "c": 2}} 

642 """ 

643 def doUpdate(d, u): 

644 if not isinstance(u, collections.abc.Mapping) or \ 644 ↛ 646line 644 didn't jump to line 646, because the condition on line 644 was never true

645 not isinstance(d, collections.abc.Mapping): 

646 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

647 for k, v in u.items(): 

648 if isinstance(v, collections.abc.Mapping): 

649 d[k] = doUpdate(d.get(k, {}), v) 

650 else: 

651 d[k] = v 

652 return d 

653 doUpdate(self._data, other) 

654 

655 def merge(self, other): 

656 """Like Config.update, but will add keys & values from other that 

657 DO NOT EXIST in self. 

658 

659 Keys and values that already exist in self will NOT be overwritten. 

660 

661 Parameters 

662 ---------- 

663 other : `dict` or `Config` 

664 Source of configuration: 

665 """ 

666 otherCopy = copy.deepcopy(other) 

667 otherCopy.update(self) 

668 self._data = otherCopy._data 

669 

670 def nameTuples(self, topLevelOnly=False): 

671 """Get tuples representing the name hierarchies of all keys. 

672 

673 The tuples returned from this method are guaranteed to be usable 

674 to access items in the configuration object. 

675 

676 Parameters 

677 ---------- 

678 topLevelOnly : `bool`, optional 

679 If False, the default, a full hierarchy of names is returned. 

680 If True, only the top level are returned. 

681 

682 Returns 

683 ------- 

684 names : `list` of `tuple` of `str` 

685 List of all names present in the `Config` where each element 

686 in the list is a `tuple` of strings representing the hierarchy. 

687 """ 

688 if topLevelOnly: 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true

689 return list((k,) for k in self) 

690 

691 def getKeysAsTuples(d, keys, base): 

692 if isinstance(d, collections.abc.Sequence): 

693 theseKeys = range(len(d)) 

694 else: 

695 theseKeys = d.keys() 

696 for key in theseKeys: 

697 val = d[key] 

698 levelKey = base + (key,) if base is not None else (key,) 

699 keys.append(levelKey) 

700 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

701 and not isinstance(val, str): 

702 getKeysAsTuples(val, keys, levelKey) 

703 keys = [] 

704 getKeysAsTuples(self._data, keys, None) 

705 return keys 

706 

707 def names(self, topLevelOnly=False, delimiter=None): 

708 """Get a delimited name of all the keys in the hierarchy. 

709 

710 The values returned from this method are guaranteed to be usable 

711 to access items in the configuration object. 

712 

713 Parameters 

714 ---------- 

715 topLevelOnly : `bool`, optional 

716 If False, the default, a full hierarchy of names is returned. 

717 If True, only the top level are returned. 

718 delimiter : `str`, optional 

719 Delimiter to use when forming the keys. If the delimiter is 

720 present in any of the keys, it will be escaped in the returned 

721 names. If `None` given a delimiter will be automatically provided. 

722 The delimiter can not be alphanumeric. 

723 

724 Returns 

725 ------- 

726 names : `list` of `str` 

727 List of all names present in the `Config`. 

728 

729 Notes 

730 ----- 

731 This is different than the built-in method `dict.keys`, which will 

732 return only the first level keys. 

733 

734 Raises 

735 ------ 

736 ValueError: 

737 The supplied delimiter is alphanumeric. 

738 """ 

739 if topLevelOnly: 

740 return list(self.keys()) 

741 

742 # Get all the tuples of hierarchical keys 

743 nameTuples = self.nameTuples() 

744 

745 if delimiter is not None and delimiter.isalnum(): 

746 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

747 

748 if delimiter is None: 

749 # Start with something, and ensure it does not need to be 

750 # escaped (it is much easier to understand if not escaped) 

751 delimiter = self._D 

752 

753 # Form big string for easy check of delimiter clash 

754 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

755 

756 # Try a delimiter and keep trying until we get something that 

757 # works. 

758 ntries = 0 

759 while delimiter in combined: 

760 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.") 

761 ntries += 1 

762 

763 if ntries > 100: 

764 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

765 

766 # try another one 

767 while True: 

768 delimiter = chr(ord(delimiter)+1) 

769 if not delimiter.isalnum(): 

770 break 

771 

772 log.debug(f"Using delimiter {delimiter!r}") 

773 

774 # Form the keys, escaping the delimiter if necessary 

775 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

776 for k in nameTuples] 

777 return strings 

778 

779 def asArray(self, name): 

780 """Get a value as an array. 

781 

782 May contain one or more elements. 

783 

784 Parameters 

785 ---------- 

786 name : `str` 

787 Key to use to retrieve value. 

788 

789 Returns 

790 ------- 

791 array : `collections.abc.Sequence` 

792 The value corresponding to name, but guaranteed to be returned 

793 as a list with at least one element. If the value is a 

794 `~collections.abc.Sequence` (and not a `str`) the value itself 

795 will be returned, else the value will be the first element. 

796 """ 

797 val = self.get(name) 

798 if isinstance(val, str): 

799 val = [val] 

800 elif not isinstance(val, collections.abc.Sequence): 

801 val = [val] 

802 return val 

803 

804 def __eq__(self, other): 

805 if isinstance(other, Config): 

806 other = other._data 

807 return self._data == other 

808 

809 def __ne__(self, other): 

810 if isinstance(other, Config): 

811 other = other._data 

812 return self._data != other 

813 

814 ####### 

815 # i/o # 

816 

817 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

818 """Writes the config to an output stream. 

819 

820 Parameters 

821 ---------- 

822 output : `IO`, optional 

823 The stream to use for output. If `None` the serialized content 

824 will be returned. 

825 format : `str`, optional 

826 The format to use for the output. Can be "yaml" or "json". 

827 

828 Returns 

829 ------- 

830 serialized : `str` or `None` 

831 If a stream was given the stream will be used and the return 

832 value will be `None`. If the stream was `None` the 

833 serialization will be returned as a string. 

834 """ 

835 if format == "yaml": 

836 return yaml.safe_dump(self._data, output, default_flow_style=False) 

837 elif format == "json": 

838 if output is not None: 

839 json.dump(self._data, output, ensure_ascii=False) 

840 return 

841 else: 

842 return json.dumps(self._data, ensure_ascii=False) 

843 raise ValueError(f"Unsupported format for Config serialization: {format}") 

844 

845 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

846 defaultFileName: str = "butler.yaml", 

847 overwrite: bool = True) -> None: 

848 """Writes the config to location pointed to by given URI. 

849 

850 Currently supports 's3' and 'file' URI schemes. 

851 

852 Parameters 

853 ---------- 

854 uri: `str` or `ButlerURI` 

855 URI of location where the Config will be written. 

856 updateFile : bool, optional 

857 If True and uri does not end on a filename with extension, will 

858 append `defaultFileName` to the target uri. True by default. 

859 defaultFileName : bool, optional 

860 The file name that will be appended to target uri if updateFile is 

861 True and uri does not end on a file with an extension. 

862 overwrite : bool, optional 

863 If True the configuration will be written even if it already 

864 exists at that location. 

865 """ 

866 # Make local copy of URI or create new one 

867 uri = ButlerURI(uri) 

868 

869 if updateFile and not uri.getExtension(): 

870 uri.updateFile(defaultFileName) 

871 

872 # Try to work out the format from the extension 

873 ext = uri.getExtension() 

874 format = ext[1:].lower() 

875 

876 uri.write(self.dump(format=format).encode(), overwrite=overwrite) 

877 self.configFile = uri 

878 

879 @staticmethod 

880 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True): 

881 """Generic helper function for updating specific config parameters. 

882 

883 Allows for named parameters to be set to new values in bulk, and 

884 for other values to be set by copying from a reference config. 

885 

886 Assumes that the supplied config is compatible with ``configType`` 

887 and will attach the updated values to the supplied config by 

888 looking for the related component key. It is assumed that 

889 ``config`` and ``full`` are from the same part of the 

890 configuration hierarchy. 

891 

892 Parameters 

893 ---------- 

894 configType : `ConfigSubset` 

895 Config type to use to extract relevant items from ``config``. 

896 config : `Config` 

897 A `Config` to update. Only the subset understood by 

898 the supplied `ConfigSubset` will be modified. Default values 

899 will not be inserted and the content will not be validated 

900 since mandatory keys are allowed to be missing until 

901 populated later by merging. 

902 full : `Config` 

903 A complete config with all defaults expanded that can be 

904 converted to a ``configType``. Read-only and will not be 

905 modified by this method. Values are read from here if 

906 ``toCopy`` is defined. 

907 

908 Repository-specific options that should not be obtained 

909 from defaults when Butler instances are constructed 

910 should be copied from ``full`` to ``config``. 

911 toUpdate : `dict`, optional 

912 A `dict` defining the keys to update and the new value to use. 

913 The keys and values can be any supported by `Config` 

914 assignment. 

915 toCopy : `tuple`, optional 

916 `tuple` of keys whose values should be copied from ``full`` 

917 into ``config``. 

918 overwrite : `bool`, optional 

919 If `False`, do not modify a value in ``config`` if the key 

920 already exists. Default is always to overwrite. 

921 

922 Raises 

923 ------ 

924 ValueError 

925 Neither ``toUpdate`` not ``toCopy`` were defined. 

926 """ 

927 if toUpdate is None and toCopy is None: 

928 raise ValueError("One of toUpdate or toCopy parameters must be set.") 

929 

930 # If this is a parent configuration then we need to ensure that 

931 # the supplied config has the relevant component key in it. 

932 # If this is a parent configuration we add in the stub entry 

933 # so that the ConfigSubset constructor will do the right thing. 

934 # We check full for this since that is guaranteed to be complete. 

935 if configType.component in full and configType.component not in config: 

936 config[configType.component] = {} 

937 

938 # Extract the part of the config we wish to update 

939 localConfig = configType(config, mergeDefaults=False, validate=False) 

940 

941 if toUpdate: 

942 for key, value in toUpdate.items(): 

943 if key in localConfig and not overwrite: 

944 log.debug("Not overriding key '%s' with value '%s' in config %s", 

945 key, value, localConfig.__class__.__name__) 

946 else: 

947 localConfig[key] = value 

948 

949 if toCopy: 

950 localFullConfig = configType(full, mergeDefaults=False) 

951 for key in toCopy: 

952 if key in localConfig and not overwrite: 

953 log.debug("Not overriding key '%s' from defaults in config %s", 

954 key, localConfig.__class__.__name__) 

955 else: 

956 localConfig[key] = localFullConfig[key] 

957 

958 # Reattach to parent if this is a child config 

959 if configType.component in config: 

960 config[configType.component] = localConfig 

961 else: 

962 config.update(localConfig) 

963 

964 def toDict(self): 

965 """Convert a `Config` to a standalone hierarchical `dict`. 

966 

967 Returns 

968 ------- 

969 d : `dict` 

970 The standalone hierarchical `dict` with any `Config` classes 

971 in the hierarchy converted to `dict`. 

972 

973 Notes 

974 ----- 

975 This can be useful when passing a Config to some code that 

976 expects native Python types. 

977 """ 

978 output = copy.deepcopy(self._data) 

979 for k, v in output.items(): 

980 if isinstance(v, Config): 

981 v = v.toDict() 

982 output[k] = v 

983 return output 

984 

985 

986class ConfigSubset(Config): 

987 """Config representing a subset of a more general configuration. 

988 

989 Subclasses define their own component and when given a configuration 

990 that includes that component, the resulting configuration only includes 

991 the subset. For example, your config might contain ``dimensions`` if it's 

992 part of a global config and that subset will be stored. If ``dimensions`` 

993 can not be found it is assumed that the entire contents of the 

994 configuration should be used. 

995 

996 Default values are read from the environment or supplied search paths 

997 using the default configuration file name specified in the subclass. 

998 This allows a configuration class to be instantiated without any 

999 additional arguments. 

1000 

1001 Additional validation can be specified to check for keys that are mandatory 

1002 in the configuration. 

1003 

1004 Parameters 

1005 ---------- 

1006 other : `Config` or `str` or `dict` 

1007 Argument specifying the configuration information as understood 

1008 by `Config` 

1009 validate : `bool`, optional 

1010 If `True` required keys will be checked to ensure configuration 

1011 consistency. 

1012 mergeDefaults : `bool`, optional 

1013 If `True` defaults will be read and the supplied config will 

1014 be combined with the defaults, with the supplied valiues taking 

1015 precedence. 

1016 searchPaths : `list` or `tuple`, optional 

1017 Explicit additional paths to search for defaults. They should 

1018 be supplied in priority order. These paths have higher priority 

1019 than those read from the environment in 

1020 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1021 the local file system or URIs, `ButlerURI`. 

1022 """ 

1023 

1024 component: ClassVar[Optional[str]] = None 

1025 """Component to use from supplied config. Can be None. If specified the 

1026 key is not required. Can be a full dot-separated path to a component. 

1027 """ 

1028 

1029 requiredKeys: ClassVar[Sequence[str]] = () 

1030 """Keys that are required to be specified in the configuration. 

1031 """ 

1032 

1033 defaultConfigFile: ClassVar[Optional[str]] = None 

1034 """Name of the file containing defaults for this config class. 

1035 """ 

1036 

1037 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1038 

1039 # Create a blank object to receive the defaults 

1040 # Once we have the defaults we then update with the external values 

1041 super().__init__() 

1042 

1043 # Create a standard Config rather than subset 

1044 externalConfig = Config(other) 

1045 

1046 # Select the part we need from it 

1047 # To simplify the use of !include we also check for the existence of 

1048 # component.component (since the included files can themselves 

1049 # include the component name) 

1050 if self.component is not None: 1050 ↛ 1059line 1050 didn't jump to line 1059, because the condition on line 1050 was never false

1051 doubled = (self.component, self.component) 

1052 # Must check for double depth first 

1053 if doubled in externalConfig: 1053 ↛ 1054line 1053 didn't jump to line 1054, because the condition on line 1053 was never true

1054 externalConfig = externalConfig[doubled] 

1055 elif self.component in externalConfig: 

1056 externalConfig._data = externalConfig._data[self.component] 

1057 

1058 # Default files read to create this configuration 

1059 self.filesRead = [] 

1060 

1061 # Assume we are not looking up child configurations 

1062 containerKey = None 

1063 

1064 # Sometimes we do not want to merge with defaults. 

1065 if mergeDefaults: 

1066 

1067 # Supplied search paths have highest priority 

1068 fullSearchPath = [] 

1069 if searchPaths: 1069 ↛ 1070line 1069 didn't jump to line 1070, because the condition on line 1069 was never true

1070 fullSearchPath.extend(searchPaths) 

1071 

1072 # Read default paths from enviroment 

1073 fullSearchPath.extend(self.defaultSearchPaths()) 

1074 

1075 # There are two places to find defaults for this particular config 

1076 # - The "defaultConfigFile" defined in the subclass 

1077 # - The class specified in the "cls" element in the config. 

1078 # Read cls after merging in case it changes. 

1079 if self.defaultConfigFile is not None: 1079 ↛ 1084line 1079 didn't jump to line 1084, because the condition on line 1079 was never false

1080 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1081 

1082 # Can have a class specification in the external config (priority) 

1083 # or from the defaults. 

1084 pytype = None 

1085 if "cls" in externalConfig: 1085 ↛ 1086line 1085 didn't jump to line 1086, because the condition on line 1085 was never true

1086 pytype = externalConfig["cls"] 

1087 elif "cls" in self: 1087 ↛ 1088line 1087 didn't jump to line 1088, because the condition on line 1087 was never true

1088 pytype = self["cls"] 

1089 

1090 if pytype is not None: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the condition on line 1090 was never true

1091 try: 

1092 cls = doImport(pytype) 

1093 except ImportError as e: 

1094 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1095 defaultsFile = cls.defaultConfigFile 

1096 if defaultsFile is not None: 

1097 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1098 

1099 # Get the container key in case we need it 

1100 try: 

1101 containerKey = cls.containerKey 

1102 except AttributeError: 

1103 pass 

1104 

1105 # Now update this object with the external values so that the external 

1106 # values always override the defaults 

1107 self.update(externalConfig) 

1108 

1109 # If this configuration has child configurations of the same 

1110 # config class, we need to expand those defaults as well. 

1111 

1112 if mergeDefaults and containerKey is not None and containerKey in self: 1112 ↛ 1113line 1112 didn't jump to line 1113, because the condition on line 1112 was never true

1113 for idx, subConfig in enumerate(self[containerKey]): 

1114 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1115 mergeDefaults=mergeDefaults, 

1116 searchPaths=searchPaths) 

1117 

1118 if validate: 

1119 self.validate() 

1120 

1121 @classmethod 

1122 def defaultSearchPaths(cls): 

1123 """Read the environment to determine search paths to use for global 

1124 defaults. 

1125 

1126 Global defaults, at lowest priority, are found in the ``config`` 

1127 directory of the butler source tree. Additional defaults can be 

1128 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1129 which is a PATH-like variable where paths at the front of the list 

1130 have priority over those later. 

1131 

1132 Returns 

1133 ------- 

1134 paths : `list` 

1135 Returns a list of paths to search. The returned order is in 

1136 priority with the highest priority paths first. The butler config 

1137 configuration resources will not be included here but will 

1138 always be searched last. 

1139 

1140 Notes 

1141 ----- 

1142 The environment variable is split on the standard ``:`` path separator. 

1143 This currently makes it incompatible with usage of URIs. 

1144 """ 

1145 # We can pick up defaults from multiple search paths 

1146 # We fill defaults by using the butler config path and then 

1147 # the config path environment variable in reverse order. 

1148 defaultsPaths = [] 

1149 

1150 if CONFIG_PATH in os.environ: 1150 ↛ 1151line 1150 didn't jump to line 1151, because the condition on line 1150 was never true

1151 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1152 defaultsPaths.extend(externalPaths) 

1153 

1154 # Add the package defaults as a resource 

1155 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1156 forceDirectory=True)) 

1157 return defaultsPaths 

1158 

1159 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1160 """Search the supplied paths, merging the configuration values 

1161 

1162 The values read will override values currently stored in the object. 

1163 Every file found in the path will be read, such that the earlier 

1164 path entries have higher priority. 

1165 

1166 Parameters 

1167 ---------- 

1168 searchPaths : `list` of `ButlerURI`, `str` 

1169 Paths to search for the supplied configFile. This path 

1170 is the priority order, such that files read from the 

1171 first path entry will be selected over those read from 

1172 a later path. Can contain `str` referring to the local file 

1173 system or a URI string. 

1174 configFile : `ButlerURI` 

1175 File to locate in path. If absolute path it will be read 

1176 directly and the search path will not be used. Can be a URI 

1177 to an explicit resource (which will ignore the search path) 

1178 which is assumed to exist. 

1179 """ 

1180 uri = ButlerURI(configFile) 

1181 if uri.isabs() and uri.exists(): 1181 ↛ 1183line 1181 didn't jump to line 1183, because the condition on line 1181 was never true

1182 # Assume this resource exists 

1183 self._updateWithOtherConfigFile(configFile) 

1184 self.filesRead.append(configFile) 

1185 else: 

1186 # Reverse order so that high priority entries 

1187 # update the object last. 

1188 for pathDir in reversed(searchPaths): 

1189 if isinstance(pathDir, (str, ButlerURI)): 1189 ↛ 1196line 1189 didn't jump to line 1196, because the condition on line 1189 was never false

1190 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1191 file = pathDir.join(configFile) 

1192 if file.exists(): 1192 ↛ 1188line 1192 didn't jump to line 1188, because the condition on line 1192 was never false

1193 self.filesRead.append(file) 

1194 self._updateWithOtherConfigFile(file) 

1195 else: 

1196 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1197 

1198 def _updateWithOtherConfigFile(self, file): 

1199 """Read in some defaults and update. 

1200 

1201 Update the configuration by reading the supplied file as a config 

1202 of this class, and merging such that these values override the 

1203 current values. Contents of the external config are not validated. 

1204 

1205 Parameters 

1206 ---------- 

1207 file : `Config`, `str`, `ButlerURI`, or `dict` 

1208 Entity that can be converted to a `ConfigSubset`. 

1209 """ 

1210 # Use this class to read the defaults so that subsetting can happen 

1211 # correctly. 

1212 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1213 self.update(externalConfig) 

1214 

1215 def validate(self): 

1216 """Check that mandatory keys are present in this configuration. 

1217 

1218 Ignored if ``requiredKeys`` is empty.""" 

1219 # Validation 

1220 missing = [k for k in self.requiredKeys if k not in self._data] 

1221 if missing: 1221 ↛ 1222line 1221 didn't jump to line 1222, because the condition on line 1221 was never true

1222 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")