Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import json 

31import logging 

32import pprint 

33import os 

34import yaml 

35import sys 

36from yaml.representer import Representer 

37import io 

38from typing import Sequence, Optional, ClassVar, IO, Union 

39 

40from lsst.utils import doImport 

41from ._butlerUri import ButlerURI 

42 

43yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

44 

45 

46# Config module logger 

47log = logging.getLogger(__name__) 

48 

49# PATH-like environment variable to use for defaults. 

50CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

51 

52try: 

53 yamlLoader = yaml.CSafeLoader 

54except AttributeError: 

55 # Not all installations have the C library 

56 yamlLoader = yaml.SafeLoader 

57 

58 

59class Loader(yamlLoader): 

60 """YAML Loader that supports file include directives 

61 

62 Uses ``!include`` directive in a YAML file to point to another 

63 YAML file to be included. The path in the include directive is relative 

64 to the file containing that directive. 

65 

66 storageClasses: !include storageClasses.yaml 

67 

68 Examples 

69 -------- 

70 >>> with open("document.yaml", "r") as f: 

71 data = yaml.load(f, Loader=Loader) 

72 

73 Notes 

74 ----- 

75 See https://davidchall.github.io/yaml-includes.html 

76 """ 

77 

78 def __init__(self, stream): 

79 super().__init__(stream) 

80 # if this is a string and not a stream we may well lack a name 

81 try: 

82 self._root = ButlerURI(stream.name) 

83 except AttributeError: 

84 # No choice but to assume a local filesystem 

85 self._root = ButlerURI("no-file.yaml") 

86 Loader.add_constructor("!include", Loader.include) 

87 

88 def include(self, node): 

89 if isinstance(node, yaml.ScalarNode): 

90 return self.extractFile(self.construct_scalar(node)) 

91 

92 elif isinstance(node, yaml.SequenceNode): 

93 result = [] 

94 for filename in self.construct_sequence(node): 

95 result.append(self.extractFile(filename)) 

96 return result 

97 

98 elif isinstance(node, yaml.MappingNode): 

99 result = {} 

100 for k, v in self.construct_mapping(node).items(): 

101 result[k] = self.extractFile(v) 

102 return result 

103 

104 else: 

105 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

106 raise yaml.constructor.ConstructorError 

107 

108 def extractFile(self, filename): 

109 # It is possible for the !include to point to an explicit URI 

110 # instead of a relative URI, therefore we first see if it is 

111 # scheme-less or not. If it has a scheme we use it directly 

112 # if it is scheme-less we use it relative to the file root. 

113 requesteduri = ButlerURI(filename, forceAbsolute=False) 

114 

115 if requesteduri.scheme: 

116 fileuri = requesteduri 

117 else: 

118 fileuri = copy.copy(self._root) 

119 fileuri.updateFile(filename) 

120 

121 log.debug("Opening YAML file via !include: %s", fileuri) 

122 

123 # Read all the data from the resource 

124 data = fileuri.read() 

125 

126 # Store the bytes into a BytesIO so we can attach a .name 

127 stream = io.BytesIO(data) 

128 stream.name = fileuri.geturl() 

129 return yaml.load(stream, Loader) 

130 

131 

132class Config(collections.abc.MutableMapping): 

133 r"""Implements a datatype that is used by `Butler` for configuration 

134 parameters. 

135 

136 It is essentially a `dict` with key/value pairs, including nested dicts 

137 (as values). In fact, it can be initialized with a `dict`. 

138 This is explained next: 

139 

140 Config extends the `dict` api so that hierarchical values may be accessed 

141 with delimited notation or as a tuple. If a string is given the delimiter 

142 is picked up from the first character in that string. For example, 

143 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

144 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

145 If the first character is alphanumeric, no delimiter will be used. 

146 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

147 Unicode characters can be used as the delimiter for distinctiveness if 

148 required. 

149 

150 If a key in the hierarchy starts with a non-alphanumeric character care 

151 should be used to ensure that either the tuple interface is used or 

152 a distinct delimiter is always given in string form. 

153 

154 Finally, the delimiter can be escaped if it is part of a key and also 

155 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

156 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

157 always better to use a different delimiter in these cases. 

158 

159 Note that adding a multi-level key implicitly creates any nesting levels 

160 that do not exist, but removing multi-level keys does not automatically 

161 remove empty nesting levels. As a result: 

162 

163 >>> c = Config() 

164 >>> c[".a.b"] = 1 

165 >>> del c[".a.b"] 

166 >>> c["a"] 

167 Config({'a': {}}) 

168 

169 Storage formats supported: 

170 

171 - yaml: read and write is supported. 

172 - json: read and write is supported but no ``!include`` directive. 

173 

174 Parameters 

175 ---------- 

176 other : `str` or `Config` or `dict` or `ButlerURI` 

177 Other source of configuration, can be: 

178 

179 - (`str` or `ButlerURI`) Treated as a URI to a config file. Must end 

180 with ".yaml". 

181 - (`Config`) Copies the other Config's values into this one. 

182 - (`dict`) Copies the values from the dict into this Config. 

183 

184 If `None` is provided an empty `Config` will be created. 

185 """ 

186 

187 _D: ClassVar[str] = "→" 

188 """Default internal delimiter to use for components in the hierarchy when 

189 constructing keys for external use (see `Config.names()`).""" 

190 

191 includeKey: ClassVar[str] = "includeConfigs" 

192 """Key used to indicate that another config should be included at this 

193 part of the hierarchy.""" 

194 

195 resourcesPackage: str = "lsst.daf.butler" 

196 """Package to search for default configuration data. The resources 

197 themselves will be within a ``configs`` resource hierarchy.""" 

198 

199 def __init__(self, other=None): 

200 self._data = {} 

201 self.configFile = None 

202 

203 if other is None: 

204 return 

205 

206 if isinstance(other, Config): 

207 self._data = copy.deepcopy(other._data) 

208 self.configFile = other.configFile 

209 elif isinstance(other, collections.abc.Mapping): 

210 self.update(other) 

211 elif isinstance(other, (str, ButlerURI)): 211 ↛ 218line 211 didn't jump to line 218, because the condition on line 211 was never false

212 # if other is a string, assume it is a file path/URI 

213 self.__initFromUri(other) 

214 self._processExplicitIncludes() 

215 else: 

216 # if the config specified by other could not be recognized raise 

217 # a runtime error. 

218 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

219 

220 def ppprint(self): 

221 """helper function for debugging, prints a config out in a readable 

222 way in the debugger. 

223 

224 use: pdb> print(myConfigObject.ppprint()) 

225 

226 Returns 

227 ------- 

228 s : `str` 

229 A prettyprint formatted string representing the config 

230 """ 

231 return pprint.pformat(self._data, indent=2, width=1) 

232 

233 def __repr__(self): 

234 return f"{type(self).__name__}({self._data!r})" 

235 

236 def __str__(self): 

237 return self.ppprint() 

238 

239 def __len__(self): 

240 return len(self._data) 

241 

242 def __iter__(self): 

243 return iter(self._data) 

244 

245 def copy(self): 

246 return type(self)(self) 

247 

248 @classmethod 

249 def fromString(cls, string: str, format: str = "yaml") -> Config: 

250 """Create a new Config instance from a serialized string. 

251 

252 Parameters 

253 ---------- 

254 string : `str` 

255 String containing content in specified format 

256 format : `str`, optional 

257 Format of the supplied string. Can be ``json`` or ``yaml``. 

258 

259 Returns 

260 ------- 

261 c : `Config` 

262 Newly-constructed Config. 

263 """ 

264 if format == "yaml": 

265 new_config = cls().__initFromYaml(string) 

266 elif format == "json": 

267 new_config = cls().__initFromJson(string) 

268 else: 

269 raise ValueError(f"Unexpected format of string: {format}") 

270 new_config._processExplicitIncludes() 

271 return new_config 

272 

273 @classmethod 

274 def fromYaml(cls, string: str) -> Config: 

275 """Create a new Config instance from a YAML string. 

276 

277 Parameters 

278 ---------- 

279 string : `str` 

280 String containing content in YAML format 

281 

282 Returns 

283 ------- 

284 c : `Config` 

285 Newly-constructed Config. 

286 """ 

287 return cls.fromString(string, format="yaml") 

288 

289 def __initFromUri(self, path: str) -> None: 

290 """Load a file from a path or an URI. 

291 

292 Parameters 

293 ---------- 

294 path : `str` 

295 Path or a URI to a persisted config file. 

296 """ 

297 uri = ButlerURI(path) 

298 ext = uri.getExtension() 

299 if ext == ".yaml": 299 ↛ 306line 299 didn't jump to line 306, because the condition on line 299 was never false

300 log.debug("Opening YAML config file: %s", uri.geturl()) 

301 content = uri.read() 

302 # Use a stream so we can name it 

303 stream = io.BytesIO(content) 

304 stream.name = uri.geturl() 

305 self.__initFromYaml(stream) 

306 elif ext == ".json": 

307 log.debug("Opening JSON config file: %s", uri.geturl()) 

308 content = uri.read() 

309 self.__initFromJson(content) 

310 else: 

311 raise RuntimeError(f"Unhandled config file type: {uri}") 

312 self.configFile = uri 

313 

314 def __initFromYaml(self, stream): 

315 """Loads a YAML config from any readable stream that contains one. 

316 

317 Parameters 

318 ---------- 

319 stream: `IO` or `str` 

320 Stream to pass to the YAML loader. Accepts anything that 

321 `yaml.load` accepts. This can include a string as well as an 

322 IO stream. 

323 

324 Raises 

325 ------ 

326 yaml.YAMLError 

327 If there is an error loading the file. 

328 """ 

329 content = yaml.load(stream, Loader=Loader) 

330 if content is None: 330 ↛ 331line 330 didn't jump to line 331, because the condition on line 330 was never true

331 content = {} 

332 self._data = content 

333 return self 

334 

335 def __initFromJson(self, stream): 

336 """Loads a JSON config from any readable stream that contains one. 

337 

338 Parameters 

339 ---------- 

340 stream: `IO` or `str` 

341 Stream to pass to the JSON loader. This can include a string as 

342 well as an IO stream. 

343 

344 Raises 

345 ------ 

346 TypeError: 

347 Raised if there is an error loading the content. 

348 """ 

349 if isinstance(stream, (bytes, str)): 

350 content = json.loads(stream) 

351 else: 

352 content = json.load(stream) 

353 if content is None: 

354 content = {} 

355 self._data = content 

356 return self 

357 

358 def _processExplicitIncludes(self): 

359 """Scan through the configuration searching for the special 

360 includeConfigs directive and process the includes.""" 

361 

362 # Search paths for config files 

363 searchPaths = [ButlerURI(os.path.curdir, forceDirectory=True)] 

364 if self.configFile is not None: 364 ↛ 372line 364 didn't jump to line 372, because the condition on line 364 was never false

365 if isinstance(self.configFile, ButlerURI): 365 ↛ 368line 365 didn't jump to line 368, because the condition on line 365 was never false

366 configDir = self.configFile.dirname() 

367 else: 

368 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

369 searchPaths.append(configDir) 

370 

371 # Ensure we know what delimiter to use 

372 names = self.nameTuples() 

373 for path in names: 

374 if path[-1] == self.includeKey: 374 ↛ 376line 374 didn't jump to line 376, because the condition on line 374 was never true

375 

376 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

377 basePath = path[:-1] 

378 

379 # Extract the includes and then delete them from the config 

380 includes = self[path] 

381 del self[path] 

382 

383 # Be consistent and convert to a list 

384 if not isinstance(includes, list): 

385 includes = [includes] 

386 

387 # Read each file assuming it is a reference to a file 

388 # The file can be relative to config file or cwd 

389 # ConfigSubset search paths are not used 

390 subConfigs = [] 

391 for fileName in includes: 

392 # Expand any shell variables -- this could be URI 

393 fileName = ButlerURI(os.path.expandvars(fileName), forceAbsolute=False) 

394 found = None 

395 if fileName.isabs(): 

396 found = fileName 

397 else: 

398 for dir in searchPaths: 

399 if isinstance(dir, ButlerURI): 

400 specific = dir.join(fileName.path) 

401 # Remote resource check might be expensive 

402 if specific.exists(): 

403 found = specific 

404 else: 

405 log.warning("Do not understand search path entry '%s' of type %s", 

406 dir, type(dir).__name__) 

407 if not found: 

408 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

409 

410 # Read the referenced Config as a Config 

411 subConfigs.append(type(self)(found)) 

412 

413 # Now we need to merge these sub configs with the current 

414 # information that was present in this node in the config 

415 # tree with precedence given to the explicit values 

416 newConfig = subConfigs.pop(0) 

417 for sc in subConfigs: 

418 newConfig.update(sc) 

419 

420 # Explicit values take precedence 

421 if not basePath: 

422 # This is an include at the root config 

423 newConfig.update(self) 

424 # Replace the current config 

425 self._data = newConfig._data 

426 else: 

427 newConfig.update(self[basePath]) 

428 # And reattach to the base config 

429 self[basePath] = newConfig 

430 

431 @staticmethod 

432 def _splitIntoKeys(key): 

433 r"""Split the argument for get/set/in into a hierarchical list. 

434 

435 Parameters 

436 ---------- 

437 key : `str` or iterable 

438 Argument given to get/set/in. If an iterable is provided it will 

439 be converted to a list. If the first character of the string 

440 is not an alphanumeric character then it will be used as the 

441 delimiter for the purposes of splitting the remainder of the 

442 string. If the delimiter is also in one of the keys then it 

443 can be escaped using ``\``. There is no default delimiter. 

444 

445 Returns 

446 ------- 

447 keys : `list` 

448 Hierarchical keys as a `list`. 

449 """ 

450 if isinstance(key, str): 

451 if not key[0].isalnum(): 451 ↛ 452line 451 didn't jump to line 452, because the condition on line 451 was never true

452 d = key[0] 

453 key = key[1:] 

454 else: 

455 return [key, ] 

456 escaped = f"\\{d}" 

457 temp = None 

458 if escaped in key: 

459 # Complain at the attempt to escape the escape 

460 doubled = fr"\{escaped}" 

461 if doubled in key: 

462 raise ValueError(f"Escaping an escaped delimiter ({doubled} in {key})" 

463 " is not yet supported.") 

464 # Replace with a character that won't be in the string 

465 temp = "\r" 

466 if temp in key or d == temp: 

467 raise ValueError(f"Can not use character {temp!r} in hierarchical key or as" 

468 " delimiter if escaping the delimiter") 

469 key = key.replace(escaped, temp) 

470 hierarchy = key.split(d) 

471 if temp: 

472 hierarchy = [h.replace(temp, d) for h in hierarchy] 

473 return hierarchy 

474 elif isinstance(key, collections.abc.Iterable): 474 ↛ 478line 474 didn't jump to line 478, because the condition on line 474 was never false

475 return list(key) 

476 else: 

477 # Not sure what this is so try it anyway 

478 return [key, ] 

479 

480 def _getKeyHierarchy(self, name): 

481 """Retrieve the key hierarchy for accessing the Config 

482 

483 Parameters 

484 ---------- 

485 name : `str` or `tuple` 

486 Delimited string or `tuple` of hierarchical keys. 

487 

488 Returns 

489 ------- 

490 hierarchy : `list` of `str` 

491 Hierarchy to use as a `list`. If the name is available directly 

492 as a key in the Config it will be used regardless of the presence 

493 of any nominal delimiter. 

494 """ 

495 if name in self._data: 

496 keys = [name, ] 

497 else: 

498 keys = self._splitIntoKeys(name) 

499 return keys 

500 

501 def _findInHierarchy(self, keys, create=False): 

502 """Look for hierarchy of keys in Config 

503 

504 Parameters 

505 ---------- 

506 keys : `list` or `tuple` 

507 Keys to search in hierarchy. 

508 create : `bool`, optional 

509 If `True`, if a part of the hierarchy does not exist, insert an 

510 empty `dict` into the hierarchy. 

511 

512 Returns 

513 ------- 

514 hierarchy : `list` 

515 List of the value corresponding to each key in the supplied 

516 hierarchy. Only keys that exist in the hierarchy will have 

517 a value. 

518 complete : `bool` 

519 `True` if the full hierarchy exists and the final element 

520 in ``hierarchy`` is the value of relevant value. 

521 """ 

522 d = self._data 

523 

524 def checkNextItem(k, d, create): 

525 """See if k is in d and if it is return the new child""" 

526 nextVal = None 

527 isThere = False 

528 if d is None: 528 ↛ 530line 528 didn't jump to line 530, because the condition on line 528 was never true

529 # We have gone past the end of the hierarchy 

530 pass 

531 elif isinstance(d, collections.abc.Sequence): 531 ↛ 536line 531 didn't jump to line 536, because the condition on line 531 was never true

532 # Check sequence first because for lists 

533 # __contains__ checks whether value is found in list 

534 # not whether the index exists in list. When we traverse 

535 # the hierarchy we are interested in the index. 

536 try: 

537 nextVal = d[int(k)] 

538 isThere = True 

539 except IndexError: 

540 pass 

541 except ValueError: 

542 isThere = k in d 

543 elif k in d: 

544 nextVal = d[k] 

545 isThere = True 

546 elif create: 546 ↛ 547line 546 didn't jump to line 547, because the condition on line 546 was never true

547 d[k] = {} 

548 nextVal = d[k] 

549 isThere = True 

550 return nextVal, isThere 

551 

552 hierarchy = [] 

553 complete = True 

554 for k in keys: 

555 d, isThere = checkNextItem(k, d, create) 

556 if isThere: 

557 hierarchy.append(d) 

558 else: 

559 complete = False 

560 break 

561 

562 return hierarchy, complete 

563 

564 def __getitem__(self, name): 

565 # Override the split for the simple case where there is an exact 

566 # match. This allows `Config.items()` to work via a simple 

567 # __iter__ implementation that returns top level keys of 

568 # self._data. 

569 keys = self._getKeyHierarchy(name) 

570 

571 hierarchy, complete = self._findInHierarchy(keys) 

572 if not complete: 572 ↛ 573line 572 didn't jump to line 573, because the condition on line 572 was never true

573 raise KeyError(f"{name} not found") 

574 data = hierarchy[-1] 

575 

576 if isinstance(data, collections.abc.Mapping): 

577 data = Config(data) 

578 # Ensure that child configs inherit the parent internal delimiter 

579 if self._D != Config._D: 579 ↛ 580line 579 didn't jump to line 580, because the condition on line 579 was never true

580 data._D = self._D 

581 return data 

582 

583 def __setitem__(self, name, value): 

584 keys = self._getKeyHierarchy(name) 

585 last = keys.pop() 

586 if isinstance(value, Config): 

587 value = copy.deepcopy(value._data) 

588 

589 hierarchy, complete = self._findInHierarchy(keys, create=True) 

590 if hierarchy: 

591 data = hierarchy[-1] 

592 else: 

593 data = self._data 

594 

595 try: 

596 data[last] = value 

597 except TypeError: 

598 data[int(last)] = value 

599 

600 def __contains__(self, key): 

601 keys = self._getKeyHierarchy(key) 

602 hierarchy, complete = self._findInHierarchy(keys) 

603 return complete 

604 

605 def __delitem__(self, key): 

606 keys = self._getKeyHierarchy(key) 

607 last = keys.pop() 

608 hierarchy, complete = self._findInHierarchy(keys) 

609 if complete: 609 ↛ 616line 609 didn't jump to line 616, because the condition on line 609 was never false

610 if hierarchy: 610 ↛ 611line 610 didn't jump to line 611, because the condition on line 610 was never true

611 data = hierarchy[-1] 

612 else: 

613 data = self._data 

614 del data[last] 

615 else: 

616 raise KeyError(f"{key} not found in Config") 

617 

618 def update(self, other): 

619 """Like dict.update, but will add or modify keys in nested dicts, 

620 instead of overwriting the nested dict entirely. 

621 

622 For example, for the given code: 

623 foo = {"a": {"b": 1}} 

624 foo.update({"a": {"c": 2}}) 

625 

626 Parameters 

627 ---------- 

628 other : `dict` or `Config` 

629 Source of configuration: 

630 

631 - If foo is a dict, then after the update foo == {"a": {"c": 2}} 

632 - But if foo is a Config, then after the update 

633 foo == {"a": {"b": 1, "c": 2}} 

634 """ 

635 def doUpdate(d, u): 

636 if not isinstance(u, collections.abc.Mapping) or \ 636 ↛ 638line 636 didn't jump to line 638, because the condition on line 636 was never true

637 not isinstance(d, collections.abc.Mapping): 

638 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

639 for k, v in u.items(): 

640 if isinstance(v, collections.abc.Mapping): 

641 d[k] = doUpdate(d.get(k, {}), v) 

642 else: 

643 d[k] = v 

644 return d 

645 doUpdate(self._data, other) 

646 

647 def merge(self, other): 

648 """Like Config.update, but will add keys & values from other that 

649 DO NOT EXIST in self. 

650 

651 Keys and values that already exist in self will NOT be overwritten. 

652 

653 Parameters 

654 ---------- 

655 other : `dict` or `Config` 

656 Source of configuration: 

657 """ 

658 otherCopy = copy.deepcopy(other) 

659 otherCopy.update(self) 

660 self._data = otherCopy._data 

661 

662 def nameTuples(self, topLevelOnly=False): 

663 """Get tuples representing the name hierarchies of all keys. 

664 

665 The tuples returned from this method are guaranteed to be usable 

666 to access items in the configuration object. 

667 

668 Parameters 

669 ---------- 

670 topLevelOnly : `bool`, optional 

671 If False, the default, a full hierarchy of names is returned. 

672 If True, only the top level are returned. 

673 

674 Returns 

675 ------- 

676 names : `list` of `tuple` of `str` 

677 List of all names present in the `Config` where each element 

678 in the list is a `tuple` of strings representing the hierarchy. 

679 """ 

680 if topLevelOnly: 680 ↛ 681line 680 didn't jump to line 681, because the condition on line 680 was never true

681 return list((k,) for k in self) 

682 

683 def getKeysAsTuples(d, keys, base): 

684 if isinstance(d, collections.abc.Sequence): 

685 theseKeys = range(len(d)) 

686 else: 

687 theseKeys = d.keys() 

688 for key in theseKeys: 

689 val = d[key] 

690 levelKey = base + (key,) if base is not None else (key,) 

691 keys.append(levelKey) 

692 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) \ 

693 and not isinstance(val, str): 

694 getKeysAsTuples(val, keys, levelKey) 

695 keys = [] 

696 getKeysAsTuples(self._data, keys, None) 

697 return keys 

698 

699 def names(self, topLevelOnly=False, delimiter=None): 

700 """Get a delimited name of all the keys in the hierarchy. 

701 

702 The values returned from this method are guaranteed to be usable 

703 to access items in the configuration object. 

704 

705 Parameters 

706 ---------- 

707 topLevelOnly : `bool`, optional 

708 If False, the default, a full hierarchy of names is returned. 

709 If True, only the top level are returned. 

710 delimiter : `str`, optional 

711 Delimiter to use when forming the keys. If the delimiter is 

712 present in any of the keys, it will be escaped in the returned 

713 names. If `None` given a delimiter will be automatically provided. 

714 The delimiter can not be alphanumeric. 

715 

716 Returns 

717 ------- 

718 names : `list` of `str` 

719 List of all names present in the `Config`. 

720 

721 Notes 

722 ----- 

723 This is different than the built-in method `dict.keys`, which will 

724 return only the first level keys. 

725 

726 Raises 

727 ------ 

728 ValueError: 

729 The supplied delimiter is alphanumeric. 

730 """ 

731 if topLevelOnly: 

732 return list(self.keys()) 

733 

734 # Get all the tuples of hierarchical keys 

735 nameTuples = self.nameTuples() 

736 

737 if delimiter is not None and delimiter.isalnum(): 

738 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

739 

740 if delimiter is None: 

741 # Start with something, and ensure it does not need to be 

742 # escaped (it is much easier to understand if not escaped) 

743 delimiter = self._D 

744 

745 # Form big string for easy check of delimiter clash 

746 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

747 

748 # Try a delimiter and keep trying until we get something that 

749 # works. 

750 ntries = 0 

751 while delimiter in combined: 

752 log.debug(f"Delimiter '{delimiter}' could not be used. Trying another.") 

753 ntries += 1 

754 

755 if ntries > 100: 

756 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

757 

758 # try another one 

759 while True: 

760 delimiter = chr(ord(delimiter)+1) 

761 if not delimiter.isalnum(): 

762 break 

763 

764 log.debug(f"Using delimiter {delimiter!r}") 

765 

766 # Form the keys, escaping the delimiter if necessary 

767 strings = [delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

768 for k in nameTuples] 

769 return strings 

770 

771 def asArray(self, name): 

772 """Get a value as an array. 

773 

774 May contain one or more elements. 

775 

776 Parameters 

777 ---------- 

778 name : `str` 

779 Key to use to retrieve value. 

780 

781 Returns 

782 ------- 

783 array : `collections.abc.Sequence` 

784 The value corresponding to name, but guaranteed to be returned 

785 as a list with at least one element. If the value is a 

786 `~collections.abc.Sequence` (and not a `str`) the value itself 

787 will be returned, else the value will be the first element. 

788 """ 

789 val = self.get(name) 

790 if isinstance(val, str): 

791 val = [val] 

792 elif not isinstance(val, collections.abc.Sequence): 

793 val = [val] 

794 return val 

795 

796 def __eq__(self, other): 

797 if isinstance(other, Config): 

798 other = other._data 

799 return self._data == other 

800 

801 def __ne__(self, other): 

802 if isinstance(other, Config): 

803 other = other._data 

804 return self._data != other 

805 

806 ####### 

807 # i/o # 

808 

809 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

810 """Writes the config to an output stream. 

811 

812 Parameters 

813 ---------- 

814 output : `IO`, optional 

815 The stream to use for output. If `None` the serialized content 

816 will be returned. 

817 format : `str`, optional 

818 The format to use for the output. Can be "yaml" or "json". 

819 

820 Returns 

821 ------- 

822 serialized : `str` or `None` 

823 If a stream was given the stream will be used and the return 

824 value will be `None`. If the stream was `None` the 

825 serialization will be returned as a string. 

826 """ 

827 if format == "yaml": 

828 return yaml.safe_dump(self._data, output, default_flow_style=False) 

829 elif format == "json": 

830 if output is not None: 

831 json.dump(self._data, output, ensure_ascii=False) 

832 return 

833 else: 

834 return json.dumps(self._data, ensure_ascii=False) 

835 raise ValueError(f"Unsupported format for Config serialization: {format}") 

836 

837 def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True, 

838 defaultFileName: str = "butler.yaml", 

839 overwrite: bool = True) -> None: 

840 """Writes the config to location pointed to by given URI. 

841 

842 Currently supports 's3' and 'file' URI schemes. 

843 

844 Parameters 

845 ---------- 

846 uri: `str` or `ButlerURI` 

847 URI of location where the Config will be written. 

848 updateFile : bool, optional 

849 If True and uri does not end on a filename with extension, will 

850 append `defaultFileName` to the target uri. True by default. 

851 defaultFileName : bool, optional 

852 The file name that will be appended to target uri if updateFile is 

853 True and uri does not end on a file with an extension. 

854 overwrite : bool, optional 

855 If True the configuration will be written even if it already 

856 exists at that location. 

857 """ 

858 # Make local copy of URI or create new one 

859 uri = ButlerURI(uri) 

860 

861 if updateFile and not uri.getExtension(): 

862 uri.updateFile(defaultFileName) 

863 

864 # Try to work out the format from the extension 

865 ext = uri.getExtension() 

866 format = ext[1:].lower() 

867 

868 uri.write(self.dump(format=format).encode(), overwrite=overwrite) 

869 self.configFile = uri 

870 

871 @staticmethod 

872 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True): 

873 """Generic helper function for updating specific config parameters. 

874 

875 Allows for named parameters to be set to new values in bulk, and 

876 for other values to be set by copying from a reference config. 

877 

878 Assumes that the supplied config is compatible with ``configType`` 

879 and will attach the updated values to the supplied config by 

880 looking for the related component key. It is assumed that 

881 ``config`` and ``full`` are from the same part of the 

882 configuration hierarchy. 

883 

884 Parameters 

885 ---------- 

886 configType : `ConfigSubset` 

887 Config type to use to extract relevant items from ``config``. 

888 config : `Config` 

889 A `Config` to update. Only the subset understood by 

890 the supplied `ConfigSubset` will be modified. Default values 

891 will not be inserted and the content will not be validated 

892 since mandatory keys are allowed to be missing until 

893 populated later by merging. 

894 full : `Config` 

895 A complete config with all defaults expanded that can be 

896 converted to a ``configType``. Read-only and will not be 

897 modified by this method. Values are read from here if 

898 ``toCopy`` is defined. 

899 

900 Repository-specific options that should not be obtained 

901 from defaults when Butler instances are constructed 

902 should be copied from ``full`` to ``config``. 

903 toUpdate : `dict`, optional 

904 A `dict` defining the keys to update and the new value to use. 

905 The keys and values can be any supported by `Config` 

906 assignment. 

907 toCopy : `tuple`, optional 

908 `tuple` of keys whose values should be copied from ``full`` 

909 into ``config``. 

910 overwrite : `bool`, optional 

911 If `False`, do not modify a value in ``config`` if the key 

912 already exists. Default is always to overwrite. 

913 

914 Raises 

915 ------ 

916 ValueError 

917 Neither ``toUpdate`` not ``toCopy`` were defined. 

918 """ 

919 if toUpdate is None and toCopy is None: 

920 raise ValueError("One of toUpdate or toCopy parameters must be set.") 

921 

922 # If this is a parent configuration then we need to ensure that 

923 # the supplied config has the relevant component key in it. 

924 # If this is a parent configuration we add in the stub entry 

925 # so that the ConfigSubset constructor will do the right thing. 

926 # We check full for this since that is guaranteed to be complete. 

927 if configType.component in full and configType.component not in config: 

928 config[configType.component] = {} 

929 

930 # Extract the part of the config we wish to update 

931 localConfig = configType(config, mergeDefaults=False, validate=False) 

932 

933 if toUpdate: 

934 for key, value in toUpdate.items(): 

935 if key in localConfig and not overwrite: 

936 log.debug("Not overriding key '%s' with value '%s' in config %s", 

937 key, value, localConfig.__class__.__name__) 

938 else: 

939 localConfig[key] = value 

940 

941 if toCopy: 

942 localFullConfig = configType(full, mergeDefaults=False) 

943 for key in toCopy: 

944 if key in localConfig and not overwrite: 

945 log.debug("Not overriding key '%s' from defaults in config %s", 

946 key, localConfig.__class__.__name__) 

947 else: 

948 localConfig[key] = localFullConfig[key] 

949 

950 # Reattach to parent if this is a child config 

951 if configType.component in config: 

952 config[configType.component] = localConfig 

953 else: 

954 config.update(localConfig) 

955 

956 def toDict(self): 

957 """Convert a `Config` to a standalone hierarchical `dict`. 

958 

959 Returns 

960 ------- 

961 d : `dict` 

962 The standalone hierarchical `dict` with any `Config` classes 

963 in the hierarchy converted to `dict`. 

964 

965 Notes 

966 ----- 

967 This can be useful when passing a Config to some code that 

968 expects native Python types. 

969 """ 

970 output = copy.deepcopy(self._data) 

971 for k, v in output.items(): 

972 if isinstance(v, Config): 

973 v = v.toDict() 

974 output[k] = v 

975 return output 

976 

977 

978class ConfigSubset(Config): 

979 """Config representing a subset of a more general configuration. 

980 

981 Subclasses define their own component and when given a configuration 

982 that includes that component, the resulting configuration only includes 

983 the subset. For example, your config might contain ``dimensions`` if it's 

984 part of a global config and that subset will be stored. If ``dimensions`` 

985 can not be found it is assumed that the entire contents of the 

986 configuration should be used. 

987 

988 Default values are read from the environment or supplied search paths 

989 using the default configuration file name specified in the subclass. 

990 This allows a configuration class to be instantiated without any 

991 additional arguments. 

992 

993 Additional validation can be specified to check for keys that are mandatory 

994 in the configuration. 

995 

996 Parameters 

997 ---------- 

998 other : `Config` or `str` or `dict` 

999 Argument specifying the configuration information as understood 

1000 by `Config` 

1001 validate : `bool`, optional 

1002 If `True` required keys will be checked to ensure configuration 

1003 consistency. 

1004 mergeDefaults : `bool`, optional 

1005 If `True` defaults will be read and the supplied config will 

1006 be combined with the defaults, with the supplied valiues taking 

1007 precedence. 

1008 searchPaths : `list` or `tuple`, optional 

1009 Explicit additional paths to search for defaults. They should 

1010 be supplied in priority order. These paths have higher priority 

1011 than those read from the environment in 

1012 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1013 the local file system or URIs, `ButlerURI`. 

1014 """ 

1015 

1016 component: ClassVar[Optional[str]] = None 

1017 """Component to use from supplied config. Can be None. If specified the 

1018 key is not required. Can be a full dot-separated path to a component. 

1019 """ 

1020 

1021 requiredKeys: ClassVar[Sequence[str]] = () 

1022 """Keys that are required to be specified in the configuration. 

1023 """ 

1024 

1025 defaultConfigFile: ClassVar[Optional[str]] = None 

1026 """Name of the file containing defaults for this config class. 

1027 """ 

1028 

1029 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1030 

1031 # Create a blank object to receive the defaults 

1032 # Once we have the defaults we then update with the external values 

1033 super().__init__() 

1034 

1035 # Create a standard Config rather than subset 

1036 externalConfig = Config(other) 

1037 

1038 # Select the part we need from it 

1039 # To simplify the use of !include we also check for the existence of 

1040 # component.component (since the included files can themselves 

1041 # include the component name) 

1042 if self.component is not None: 1042 ↛ 1051line 1042 didn't jump to line 1051, because the condition on line 1042 was never false

1043 doubled = (self.component, self.component) 

1044 # Must check for double depth first 

1045 if doubled in externalConfig: 1045 ↛ 1046line 1045 didn't jump to line 1046, because the condition on line 1045 was never true

1046 externalConfig = externalConfig[doubled] 

1047 elif self.component in externalConfig: 

1048 externalConfig._data = externalConfig._data[self.component] 

1049 

1050 # Default files read to create this configuration 

1051 self.filesRead = [] 

1052 

1053 # Assume we are not looking up child configurations 

1054 containerKey = None 

1055 

1056 # Sometimes we do not want to merge with defaults. 

1057 if mergeDefaults: 

1058 

1059 # Supplied search paths have highest priority 

1060 fullSearchPath = [] 

1061 if searchPaths: 1061 ↛ 1062line 1061 didn't jump to line 1062, because the condition on line 1061 was never true

1062 fullSearchPath.extend(searchPaths) 

1063 

1064 # Read default paths from enviroment 

1065 fullSearchPath.extend(self.defaultSearchPaths()) 

1066 

1067 # There are two places to find defaults for this particular config 

1068 # - The "defaultConfigFile" defined in the subclass 

1069 # - The class specified in the "cls" element in the config. 

1070 # Read cls after merging in case it changes. 

1071 if self.defaultConfigFile is not None: 1071 ↛ 1076line 1071 didn't jump to line 1076, because the condition on line 1071 was never false

1072 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1073 

1074 # Can have a class specification in the external config (priority) 

1075 # or from the defaults. 

1076 pytype = None 

1077 if "cls" in externalConfig: 1077 ↛ 1078line 1077 didn't jump to line 1078, because the condition on line 1077 was never true

1078 pytype = externalConfig["cls"] 

1079 elif "cls" in self: 1079 ↛ 1080line 1079 didn't jump to line 1080, because the condition on line 1079 was never true

1080 pytype = self["cls"] 

1081 

1082 if pytype is not None: 1082 ↛ 1083line 1082 didn't jump to line 1083, because the condition on line 1082 was never true

1083 try: 

1084 cls = doImport(pytype) 

1085 except ImportError as e: 

1086 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1087 defaultsFile = cls.defaultConfigFile 

1088 if defaultsFile is not None: 

1089 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1090 

1091 # Get the container key in case we need it 

1092 try: 

1093 containerKey = cls.containerKey 

1094 except AttributeError: 

1095 pass 

1096 

1097 # Now update this object with the external values so that the external 

1098 # values always override the defaults 

1099 self.update(externalConfig) 

1100 

1101 # If this configuration has child configurations of the same 

1102 # config class, we need to expand those defaults as well. 

1103 

1104 if mergeDefaults and containerKey is not None and containerKey in self: 1104 ↛ 1105line 1104 didn't jump to line 1105, because the condition on line 1104 was never true

1105 for idx, subConfig in enumerate(self[containerKey]): 

1106 self[containerKey, idx] = type(self)(other=subConfig, validate=validate, 

1107 mergeDefaults=mergeDefaults, 

1108 searchPaths=searchPaths) 

1109 

1110 if validate: 

1111 self.validate() 

1112 

1113 @classmethod 

1114 def defaultSearchPaths(cls): 

1115 """Read the environment to determine search paths to use for global 

1116 defaults. 

1117 

1118 Global defaults, at lowest priority, are found in the ``config`` 

1119 directory of the butler source tree. Additional defaults can be 

1120 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1121 which is a PATH-like variable where paths at the front of the list 

1122 have priority over those later. 

1123 

1124 Returns 

1125 ------- 

1126 paths : `list` 

1127 Returns a list of paths to search. The returned order is in 

1128 priority with the highest priority paths first. The butler config 

1129 configuration resources will not be included here but will 

1130 always be searched last. 

1131 

1132 Notes 

1133 ----- 

1134 The environment variable is split on the standard ``:`` path separator. 

1135 This currently makes it incompatible with usage of URIs. 

1136 """ 

1137 # We can pick up defaults from multiple search paths 

1138 # We fill defaults by using the butler config path and then 

1139 # the config path environment variable in reverse order. 

1140 defaultsPaths = [] 

1141 

1142 if CONFIG_PATH in os.environ: 1142 ↛ 1143line 1142 didn't jump to line 1143, because the condition on line 1142 was never true

1143 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1144 defaultsPaths.extend(externalPaths) 

1145 

1146 # Add the package defaults as a resource 

1147 defaultsPaths.append(ButlerURI(f"resource://{cls.resourcesPackage}/configs", 

1148 forceDirectory=True)) 

1149 return defaultsPaths 

1150 

1151 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1152 """Search the supplied paths, merging the configuration values 

1153 

1154 The values read will override values currently stored in the object. 

1155 Every file found in the path will be read, such that the earlier 

1156 path entries have higher priority. 

1157 

1158 Parameters 

1159 ---------- 

1160 searchPaths : `list` of `ButlerURI`, `str` 

1161 Paths to search for the supplied configFile. This path 

1162 is the priority order, such that files read from the 

1163 first path entry will be selected over those read from 

1164 a later path. Can contain `str` referring to the local file 

1165 system or a URI string. 

1166 configFile : `ButlerURI` 

1167 File to locate in path. If absolute path it will be read 

1168 directly and the search path will not be used. Can be a URI 

1169 to an explicit resource (which will ignore the search path) 

1170 which is assumed to exist. 

1171 """ 

1172 uri = ButlerURI(configFile) 

1173 if uri.isabs() and uri.exists(): 1173 ↛ 1175line 1173 didn't jump to line 1175, because the condition on line 1173 was never true

1174 # Assume this resource exists 

1175 self._updateWithOtherConfigFile(configFile) 

1176 self.filesRead.append(configFile) 

1177 else: 

1178 # Reverse order so that high priority entries 

1179 # update the object last. 

1180 for pathDir in reversed(searchPaths): 

1181 if isinstance(pathDir, (str, ButlerURI)): 1181 ↛ 1188line 1181 didn't jump to line 1188, because the condition on line 1181 was never false

1182 pathDir = ButlerURI(pathDir, forceDirectory=True) 

1183 file = pathDir.join(configFile) 

1184 if file.exists(): 1184 ↛ 1180line 1184 didn't jump to line 1180, because the condition on line 1184 was never false

1185 self.filesRead.append(file) 

1186 self._updateWithOtherConfigFile(file) 

1187 else: 

1188 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1189 

1190 def _updateWithOtherConfigFile(self, file): 

1191 """Read in some defaults and update. 

1192 

1193 Update the configuration by reading the supplied file as a config 

1194 of this class, and merging such that these values override the 

1195 current values. Contents of the external config are not validated. 

1196 

1197 Parameters 

1198 ---------- 

1199 file : `Config`, `str`, `ButlerURI`, or `dict` 

1200 Entity that can be converted to a `ConfigSubset`. 

1201 """ 

1202 # Use this class to read the defaults so that subsetting can happen 

1203 # correctly. 

1204 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1205 self.update(externalConfig) 

1206 

1207 def validate(self): 

1208 """Check that mandatory keys are present in this configuration. 

1209 

1210 Ignored if ``requiredKeys`` is empty.""" 

1211 # Validation 

1212 missing = [k for k in self.requiredKeys if k not in self._data] 

1213 if missing: 1213 ↛ 1214line 1213 didn't jump to line 1214, because the condition on line 1213 was never true

1214 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")