Coverage for python/lsst/daf/butler/core/config.py: 44%

482 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-10-26 15:15 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""Configuration control.""" 

25 

26__all__ = ("Config", "ConfigSubset") 

27 

28import collections 

29import copy 

30import io 

31import json 

32import logging 

33import os 

34import pprint 

35import sys 

36from pathlib import Path 

37from typing import IO, Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Union 

38 

39import yaml 

40from lsst.resources import ResourcePath, ResourcePathExpression 

41from lsst.utils import doImport 

42from yaml.representer import Representer 

43 

44yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

45 

46 

47# Config module logger 

48log = logging.getLogger(__name__) 

49 

50# PATH-like environment variable to use for defaults. 

51CONFIG_PATH = "DAF_BUTLER_CONFIG_PATH" 

52 

53try: 

54 yamlLoader = yaml.CSafeLoader 

55except AttributeError: 

56 # Not all installations have the C library 

57 # (but assume for mypy's sake that they're the same) 

58 yamlLoader = yaml.SafeLoader # type: ignore 

59 

60 

61def _doUpdate(d, u): 

62 if not isinstance(u, collections.abc.Mapping) or not isinstance(d, collections.abc.MutableMapping): 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true

63 raise RuntimeError("Only call update with Mapping, not {}".format(type(d))) 

64 for k, v in u.items(): 

65 if isinstance(v, collections.abc.Mapping): 

66 d[k] = _doUpdate(d.get(k, {}), v) 

67 else: 

68 d[k] = v 

69 return d 

70 

71 

72def _checkNextItem(k, d, create, must_be_dict): 

73 """See if k is in d and if it is return the new child.""" 

74 nextVal = None 

75 isThere = False 

76 if d is None: 76 ↛ 78line 76 didn't jump to line 78, because the condition on line 76 was never true

77 # We have gone past the end of the hierarchy 

78 pass 

79 elif not must_be_dict and isinstance(d, collections.abc.Sequence): 79 ↛ 84line 79 didn't jump to line 84, because the condition on line 79 was never true

80 # Check for Sequence first because for lists 

81 # __contains__ checks whether value is found in list 

82 # not whether the index exists in list. When we traverse 

83 # the hierarchy we are interested in the index. 

84 try: 

85 nextVal = d[int(k)] 

86 isThere = True 

87 except IndexError: 

88 pass 

89 except ValueError: 

90 isThere = k in d 

91 elif k in d: 

92 nextVal = d[k] 

93 isThere = True 

94 elif create: 94 ↛ 95line 94 didn't jump to line 95, because the condition on line 94 was never true

95 d[k] = {} 

96 nextVal = d[k] 

97 isThere = True 

98 

99 return nextVal, isThere 

100 

101 

102class Loader(yamlLoader): 

103 """YAML Loader that supports file include directives. 

104 

105 Uses ``!include`` directive in a YAML file to point to another 

106 YAML file to be included. The path in the include directive is relative 

107 to the file containing that directive. 

108 

109 storageClasses: !include storageClasses.yaml 

110 

111 Examples 

112 -------- 

113 >>> with open("document.yaml", "r") as f: 

114 data = yaml.load(f, Loader=Loader) 

115 

116 Notes 

117 ----- 

118 See https://davidchall.github.io/yaml-includes.html 

119 """ 

120 

121 def __init__(self, stream): 

122 super().__init__(stream) 

123 # if this is a string and not a stream we may well lack a name 

124 try: 

125 self._root = ResourcePath(stream.name) 

126 except AttributeError: 

127 # No choice but to assume a local filesystem 

128 self._root = ResourcePath("no-file.yaml") 

129 Loader.add_constructor("!include", Loader.include) 

130 

131 def include(self, node): 

132 result: Union[List[Any], Dict[str, Any]] 

133 if isinstance(node, yaml.ScalarNode): 

134 return self.extractFile(self.construct_scalar(node)) 

135 

136 elif isinstance(node, yaml.SequenceNode): 

137 result = [] 

138 for filename in self.construct_sequence(node): 

139 result.append(self.extractFile(filename)) 

140 return result 

141 

142 elif isinstance(node, yaml.MappingNode): 

143 result = {} 

144 for k, v in self.construct_mapping(node).items(): 

145 result[k] = self.extractFile(v) 

146 return result 

147 

148 else: 

149 print("Error:: unrecognised node type in !include statement", file=sys.stderr) 

150 raise yaml.constructor.ConstructorError 

151 

152 def extractFile(self, filename): 

153 # It is possible for the !include to point to an explicit URI 

154 # instead of a relative URI, therefore we first see if it is 

155 # scheme-less or not. If it has a scheme we use it directly 

156 # if it is scheme-less we use it relative to the file root. 

157 requesteduri = ResourcePath(filename, forceAbsolute=False) 

158 

159 if requesteduri.scheme: 

160 fileuri = requesteduri 

161 else: 

162 fileuri = self._root.updatedFile(filename) 

163 

164 log.debug("Opening YAML file via !include: %s", fileuri) 

165 

166 # Read all the data from the resource 

167 data = fileuri.read() 

168 

169 # Store the bytes into a BytesIO so we can attach a .name 

170 stream = io.BytesIO(data) 

171 stream.name = fileuri.geturl() 

172 return yaml.load(stream, Loader) 

173 

174 

175class Config(collections.abc.MutableMapping): 

176 r"""Implements a datatype that is used by `Butler` for configuration. 

177 

178 It is essentially a `dict` with key/value pairs, including nested dicts 

179 (as values). In fact, it can be initialized with a `dict`. 

180 This is explained next: 

181 

182 Config extends the `dict` api so that hierarchical values may be accessed 

183 with delimited notation or as a tuple. If a string is given the delimiter 

184 is picked up from the first character in that string. For example, 

185 ``foo.getValue(".a.b.c")``, ``foo["a"]["b"]["c"]``, ``foo["a", "b", "c"]``, 

186 ``foo[".a.b.c"]``, and ``foo["/a/b/c"]`` all achieve the same outcome. 

187 If the first character is alphanumeric, no delimiter will be used. 

188 ``foo["a.b.c"]`` will be a single key ``a.b.c`` as will ``foo[":a.b.c"]``. 

189 Unicode characters can be used as the delimiter for distinctiveness if 

190 required. 

191 

192 If a key in the hierarchy starts with a non-alphanumeric character care 

193 should be used to ensure that either the tuple interface is used or 

194 a distinct delimiter is always given in string form. 

195 

196 Finally, the delimiter can be escaped if it is part of a key and also 

197 has to be used as a delimiter. For example, ``foo[r".a.b\.c"]`` results in 

198 a two element hierarchy of ``a`` and ``b.c``. For hard-coded strings it is 

199 always better to use a different delimiter in these cases. 

200 

201 Note that adding a multi-level key implicitly creates any nesting levels 

202 that do not exist, but removing multi-level keys does not automatically 

203 remove empty nesting levels. As a result: 

204 

205 >>> c = Config() 

206 >>> c[".a.b"] = 1 

207 >>> del c[".a.b"] 

208 >>> c["a"] 

209 Config({'a': {}}) 

210 

211 Storage formats supported: 

212 

213 - yaml: read and write is supported. 

214 - json: read and write is supported but no ``!include`` directive. 

215 

216 Parameters 

217 ---------- 

218 other : `lsst.resources.ResourcePath` or `Config` or `dict` 

219 Other source of configuration, can be: 

220 

221 - (`lsst.resources.ResourcePathExpression`) 

222 Treated as a URI to a config file. Must end with ".yaml". 

223 - (`Config`) Copies the other Config's values into this one. 

224 - (`dict`) Copies the values from the dict into this Config. 

225 

226 If `None` is provided an empty `Config` will be created. 

227 """ 

228 

229 _D: str = "→" 

230 """Default internal delimiter to use for components in the hierarchy when 

231 constructing keys for external use (see `Config.names()`).""" 

232 

233 includeKey: ClassVar[str] = "includeConfigs" 

234 """Key used to indicate that another config should be included at this 

235 part of the hierarchy.""" 

236 

237 resourcesPackage: str = "lsst.daf.butler" 

238 """Package to search for default configuration data. The resources 

239 themselves will be within a ``configs`` resource hierarchy.""" 

240 

241 def __init__(self, other=None): 

242 self._data: Dict[str, Any] = {} 

243 self.configFile = None 

244 

245 if other is None: 

246 return 

247 

248 if isinstance(other, Config): 

249 self._data = copy.deepcopy(other._data) 

250 self.configFile = other.configFile 

251 elif isinstance(other, (dict, collections.abc.Mapping)): 

252 # In most cases we have a dict, and it's more efficient 

253 # to check for a dict instance before checking the generic mapping. 

254 self.update(other) 

255 elif isinstance(other, (str, ResourcePath, Path)): 255 ↛ 262line 255 didn't jump to line 262, because the condition on line 255 was never false

256 # if other is a string, assume it is a file path/URI 

257 self.__initFromUri(other) 

258 self._processExplicitIncludes() 

259 else: 

260 # if the config specified by other could not be recognized raise 

261 # a runtime error. 

262 raise RuntimeError(f"A Config could not be loaded from other: {other}") 

263 

264 def ppprint(self): 

265 """Return config as formatted readable string. 

266 

267 Examples 

268 -------- 

269 use: ``pdb> print(myConfigObject.ppprint())`` 

270 

271 Returns 

272 ------- 

273 s : `str` 

274 A prettyprint formatted string representing the config 

275 """ 

276 return pprint.pformat(self._data, indent=2, width=1) 

277 

278 def __repr__(self): 

279 return f"{type(self).__name__}({self._data!r})" 

280 

281 def __str__(self): 

282 return self.ppprint() 

283 

284 def __len__(self): 

285 return len(self._data) 

286 

287 def __iter__(self): 

288 return iter(self._data) 

289 

290 def copy(self): 

291 return type(self)(self) 

292 

293 @classmethod 

294 def fromString(cls, string: str, format: str = "yaml") -> Config: 

295 """Create a new Config instance from a serialized string. 

296 

297 Parameters 

298 ---------- 

299 string : `str` 

300 String containing content in specified format 

301 format : `str`, optional 

302 Format of the supplied string. Can be ``json`` or ``yaml``. 

303 

304 Returns 

305 ------- 

306 c : `Config` 

307 Newly-constructed Config. 

308 """ 

309 if format == "yaml": 

310 new_config = cls().__initFromYaml(string) 

311 elif format == "json": 

312 new_config = cls().__initFromJson(string) 

313 else: 

314 raise ValueError(f"Unexpected format of string: {format}") 

315 new_config._processExplicitIncludes() 

316 return new_config 

317 

318 @classmethod 

319 def fromYaml(cls, string: str) -> Config: 

320 """Create a new Config instance from a YAML string. 

321 

322 Parameters 

323 ---------- 

324 string : `str` 

325 String containing content in YAML format 

326 

327 Returns 

328 ------- 

329 c : `Config` 

330 Newly-constructed Config. 

331 """ 

332 return cls.fromString(string, format="yaml") 

333 

334 def __initFromUri(self, path: ResourcePathExpression) -> None: 

335 """Load a file from a path or an URI. 

336 

337 Parameters 

338 ---------- 

339 path : `lsst.resources.ResourcePathExpression` 

340 Path or a URI to a persisted config file. 

341 """ 

342 uri = ResourcePath(path) 

343 ext = uri.getExtension() 

344 if ext == ".yaml": 344 ↛ 351line 344 didn't jump to line 351, because the condition on line 344 was never false

345 log.debug("Opening YAML config file: %s", uri.geturl()) 

346 content = uri.read() 

347 # Use a stream so we can name it 

348 stream = io.BytesIO(content) 

349 stream.name = uri.geturl() 

350 self.__initFromYaml(stream) 

351 elif ext == ".json": 

352 log.debug("Opening JSON config file: %s", uri.geturl()) 

353 content = uri.read() 

354 self.__initFromJson(content) 

355 else: 

356 # This URI does not have a valid extension. It might be because 

357 # we ended up with a directory and not a file. Before we complain 

358 # about an extension, do an existence check. No need to do 

359 # the (possibly expensive) existence check in the default code 

360 # path above because we will find out soon enough that the file 

361 # is not there. 

362 if not uri.exists(): 

363 raise FileNotFoundError(f"Config location {uri} does not exist.") 

364 raise RuntimeError(f"The Config URI does not have a supported extension: {uri}") 

365 self.configFile = uri 

366 

367 def __initFromYaml(self, stream): 

368 """Load a YAML config from any readable stream that contains one. 

369 

370 Parameters 

371 ---------- 

372 stream: `IO` or `str` 

373 Stream to pass to the YAML loader. Accepts anything that 

374 `yaml.load` accepts. This can include a string as well as an 

375 IO stream. 

376 

377 Raises 

378 ------ 

379 yaml.YAMLError 

380 If there is an error loading the file. 

381 """ 

382 content = yaml.load(stream, Loader=Loader) 

383 if content is None: 383 ↛ 384line 383 didn't jump to line 384, because the condition on line 383 was never true

384 content = {} 

385 self._data = content 

386 return self 

387 

388 def __initFromJson(self, stream): 

389 """Load a JSON config from any readable stream that contains one. 

390 

391 Parameters 

392 ---------- 

393 stream: `IO` or `str` 

394 Stream to pass to the JSON loader. This can include a string as 

395 well as an IO stream. 

396 

397 Raises 

398 ------ 

399 TypeError: 

400 Raised if there is an error loading the content. 

401 """ 

402 if isinstance(stream, (bytes, str)): 

403 content = json.loads(stream) 

404 else: 

405 content = json.load(stream) 

406 if content is None: 

407 content = {} 

408 self._data = content 

409 return self 

410 

411 def _processExplicitIncludes(self): 

412 """Scan through the configuration searching for the special includes. 

413 

414 Looks for ``includeConfigs`` directive and processes the includes. 

415 """ 

416 # Search paths for config files 

417 searchPaths = [ResourcePath(os.path.curdir, forceDirectory=True)] 

418 if self.configFile is not None: 418 ↛ 426line 418 didn't jump to line 426, because the condition on line 418 was never false

419 if isinstance(self.configFile, ResourcePath): 419 ↛ 422line 419 didn't jump to line 422, because the condition on line 419 was never false

420 configDir = self.configFile.dirname() 

421 else: 

422 raise RuntimeError(f"Unexpected type for config file: {self.configFile}") 

423 searchPaths.append(configDir) 

424 

425 # Ensure we know what delimiter to use 

426 names = self.nameTuples() 

427 for path in names: 

428 if path[-1] == self.includeKey: 428 ↛ 429line 428 didn't jump to line 429, because the condition on line 428 was never true

429 log.debug("Processing file include directive at %s", self._D + self._D.join(path)) 

430 basePath = path[:-1] 

431 

432 # Extract the includes and then delete them from the config 

433 includes = self[path] 

434 del self[path] 

435 

436 # Be consistent and convert to a list 

437 if not isinstance(includes, list): 

438 includes = [includes] 

439 

440 # Read each file assuming it is a reference to a file 

441 # The file can be relative to config file or cwd 

442 # ConfigSubset search paths are not used 

443 subConfigs = [] 

444 for fileName in includes: 

445 # Expand any shell variables -- this could be URI 

446 fileName = ResourcePath(os.path.expandvars(fileName), forceAbsolute=False) 

447 found = None 

448 if fileName.isabs(): 

449 found = fileName 

450 else: 

451 for dir in searchPaths: 

452 if isinstance(dir, ResourcePath): 

453 specific = dir.join(fileName.path) 

454 # Remote resource check might be expensive 

455 if specific.exists(): 

456 found = specific 

457 else: 

458 log.warning( 

459 "Do not understand search path entry '%s' of type %s", 

460 dir, 

461 type(dir).__name__, 

462 ) 

463 if not found: 

464 raise RuntimeError(f"Unable to find referenced include file: {fileName}") 

465 

466 # Read the referenced Config as a Config 

467 subConfigs.append(type(self)(found)) 

468 

469 # Now we need to merge these sub configs with the current 

470 # information that was present in this node in the config 

471 # tree with precedence given to the explicit values 

472 newConfig = subConfigs.pop(0) 

473 for sc in subConfigs: 

474 newConfig.update(sc) 

475 

476 # Explicit values take precedence 

477 if not basePath: 

478 # This is an include at the root config 

479 newConfig.update(self) 

480 # Replace the current config 

481 self._data = newConfig._data 

482 else: 

483 newConfig.update(self[basePath]) 

484 # And reattach to the base config 

485 self[basePath] = newConfig 

486 

487 @staticmethod 

488 def _splitIntoKeys(key): 

489 r"""Split the argument for get/set/in into a hierarchical list. 

490 

491 Parameters 

492 ---------- 

493 key : `str` or iterable 

494 Argument given to get/set/in. If an iterable is provided it will 

495 be converted to a list. If the first character of the string 

496 is not an alphanumeric character then it will be used as the 

497 delimiter for the purposes of splitting the remainder of the 

498 string. If the delimiter is also in one of the keys then it 

499 can be escaped using ``\``. There is no default delimiter. 

500 

501 Returns 

502 ------- 

503 keys : `list` 

504 Hierarchical keys as a `list`. 

505 """ 

506 if isinstance(key, str): 

507 if not key[0].isalnum(): 507 ↛ 508line 507 didn't jump to line 508, because the condition on line 507 was never true

508 d = key[0] 

509 key = key[1:] 

510 else: 

511 return [ 

512 key, 

513 ] 

514 escaped = f"\\{d}" 

515 temp = None 

516 if escaped in key: 

517 # Complain at the attempt to escape the escape 

518 doubled = rf"\{escaped}" 

519 if doubled in key: 

520 raise ValueError( 

521 f"Escaping an escaped delimiter ({doubled} in {key}) is not yet supported." 

522 ) 

523 # Replace with a character that won't be in the string 

524 temp = "\r" 

525 if temp in key or d == temp: 

526 raise ValueError( 

527 f"Can not use character {temp!r} in hierarchical key or as" 

528 " delimiter if escaping the delimiter" 

529 ) 

530 key = key.replace(escaped, temp) 

531 hierarchy = key.split(d) 

532 if temp: 

533 hierarchy = [h.replace(temp, d) for h in hierarchy] 

534 return hierarchy 

535 elif isinstance(key, collections.abc.Iterable): 535 ↛ 539line 535 didn't jump to line 539, because the condition on line 535 was never false

536 return list(key) 

537 else: 

538 # Not sure what this is so try it anyway 

539 return [ 

540 key, 

541 ] 

542 

543 def _getKeyHierarchy(self, name): 

544 """Retrieve the key hierarchy for accessing the Config. 

545 

546 Parameters 

547 ---------- 

548 name : `str` or `tuple` 

549 Delimited string or `tuple` of hierarchical keys. 

550 

551 Returns 

552 ------- 

553 hierarchy : `list` of `str` 

554 Hierarchy to use as a `list`. If the name is available directly 

555 as a key in the Config it will be used regardless of the presence 

556 of any nominal delimiter. 

557 """ 

558 if name in self._data: 

559 keys = [ 

560 name, 

561 ] 

562 else: 

563 keys = self._splitIntoKeys(name) 

564 return keys 

565 

566 def _findInHierarchy(self, keys, create=False): 

567 """Look for hierarchy of keys in Config. 

568 

569 Parameters 

570 ---------- 

571 keys : `list` or `tuple` 

572 Keys to search in hierarchy. 

573 create : `bool`, optional 

574 If `True`, if a part of the hierarchy does not exist, insert an 

575 empty `dict` into the hierarchy. 

576 

577 Returns 

578 ------- 

579 hierarchy : `list` 

580 List of the value corresponding to each key in the supplied 

581 hierarchy. Only keys that exist in the hierarchy will have 

582 a value. 

583 complete : `bool` 

584 `True` if the full hierarchy exists and the final element 

585 in ``hierarchy`` is the value of relevant value. 

586 """ 

587 d = self._data 

588 

589 # For the first key, d must be a dict so it is a waste 

590 # of time to check for a sequence. 

591 must_be_dict = True 

592 

593 hierarchy = [] 

594 complete = True 

595 for k in keys: 

596 d, isThere = _checkNextItem(k, d, create, must_be_dict) 

597 if isThere: 

598 hierarchy.append(d) 

599 else: 

600 complete = False 

601 break 

602 # Second time round it might be a sequence. 

603 must_be_dict = False 

604 

605 return hierarchy, complete 

606 

607 def __getitem__(self, name): 

608 # Override the split for the simple case where there is an exact 

609 # match. This allows `Config.items()` to work via a simple 

610 # __iter__ implementation that returns top level keys of 

611 # self._data. 

612 

613 # If the name matches a key in the top-level hierarchy, bypass 

614 # all further cleverness. 

615 found_directly = False 

616 try: 

617 data = self._data[name] 

618 found_directly = True 

619 except KeyError: 

620 pass 

621 

622 if not found_directly: 622 ↛ 623line 622 didn't jump to line 623, because the condition on line 622 was never true

623 keys = self._getKeyHierarchy(name) 

624 

625 hierarchy, complete = self._findInHierarchy(keys) 

626 if not complete: 

627 raise KeyError(f"{name} not found") 

628 data = hierarchy[-1] 

629 

630 # In most cases we have a dict, and it's more efficient 

631 # to check for a dict instance before checking the generic mapping. 

632 if isinstance(data, (dict, collections.abc.Mapping)): 

633 data = Config(data) 

634 # Ensure that child configs inherit the parent internal delimiter 

635 if self._D != Config._D: 635 ↛ 636line 635 didn't jump to line 636, because the condition on line 635 was never true

636 data._D = self._D 

637 return data 

638 

639 def __setitem__(self, name, value): 

640 keys = self._getKeyHierarchy(name) 

641 last = keys.pop() 

642 if isinstance(value, Config): 

643 value = copy.deepcopy(value._data) 

644 

645 hierarchy, complete = self._findInHierarchy(keys, create=True) 

646 if hierarchy: 

647 data = hierarchy[-1] 

648 else: 

649 data = self._data 

650 

651 try: 

652 data[last] = value 

653 except TypeError: 

654 data[int(last)] = value 

655 

656 def __contains__(self, key): 

657 keys = self._getKeyHierarchy(key) 

658 hierarchy, complete = self._findInHierarchy(keys) 

659 return complete 

660 

661 def __delitem__(self, key): 

662 keys = self._getKeyHierarchy(key) 

663 last = keys.pop() 

664 hierarchy, complete = self._findInHierarchy(keys) 

665 if complete: 665 ↛ 672line 665 didn't jump to line 672, because the condition on line 665 was never false

666 if hierarchy: 666 ↛ 667line 666 didn't jump to line 667, because the condition on line 666 was never true

667 data = hierarchy[-1] 

668 else: 

669 data = self._data 

670 del data[last] 

671 else: 

672 raise KeyError(f"{key} not found in Config") 

673 

674 def update(self, other): 

675 """Update config from other `Config` or `dict`. 

676 

677 Like `dict.update()`, but will add or modify keys in nested dicts, 

678 instead of overwriting the nested dict entirely. 

679 

680 Parameters 

681 ---------- 

682 other : `dict` or `Config` 

683 Source of configuration: 

684 

685 Examples 

686 -------- 

687 >>> c = Config({"a": {"b": 1}}) 

688 >>> c.update({"a": {"c": 2}}) 

689 >>> print(c) 

690 {'a': {'b': 1, 'c': 2}} 

691 

692 >>> foo = {"a": {"b": 1}} 

693 >>> foo.update({"a": {"c": 2}}) 

694 >>> print(foo) 

695 {'a': {'c': 2}} 

696 """ 

697 _doUpdate(self._data, other) 

698 

699 def merge(self, other): 

700 """Merge another Config into this one. 

701 

702 Like `Config.update()`, but will add keys & values from other that 

703 DO NOT EXIST in self. 

704 

705 Keys and values that already exist in self will NOT be overwritten. 

706 

707 Parameters 

708 ---------- 

709 other : `dict` or `Config` 

710 Source of configuration: 

711 """ 

712 if not isinstance(other, collections.abc.Mapping): 

713 raise TypeError(f"Can only merge a Mapping into a Config, not {type(other)}") 

714 

715 # Convert the supplied mapping to a Config for consistency 

716 # This will do a deepcopy if it is already a Config 

717 otherCopy = Config(other) 

718 otherCopy.update(self) 

719 self._data = otherCopy._data 

720 

721 def nameTuples(self, topLevelOnly=False): 

722 """Get tuples representing the name hierarchies of all keys. 

723 

724 The tuples returned from this method are guaranteed to be usable 

725 to access items in the configuration object. 

726 

727 Parameters 

728 ---------- 

729 topLevelOnly : `bool`, optional 

730 If False, the default, a full hierarchy of names is returned. 

731 If True, only the top level are returned. 

732 

733 Returns 

734 ------- 

735 names : `list` of `tuple` of `str` 

736 List of all names present in the `Config` where each element 

737 in the list is a `tuple` of strings representing the hierarchy. 

738 """ 

739 if topLevelOnly: 739 ↛ 740line 739 didn't jump to line 740, because the condition on line 739 was never true

740 return list((k,) for k in self) 

741 

742 def getKeysAsTuples(d, keys, base): 

743 if isinstance(d, collections.abc.Sequence): 

744 theseKeys = range(len(d)) 

745 else: 

746 theseKeys = d.keys() 

747 for key in theseKeys: 

748 val = d[key] 

749 levelKey = base + (key,) if base is not None else (key,) 

750 keys.append(levelKey) 

751 if isinstance(val, (collections.abc.Mapping, collections.abc.Sequence)) and not isinstance( 

752 val, str 

753 ): 

754 getKeysAsTuples(val, keys, levelKey) 

755 

756 keys: List[Tuple[str, ...]] = [] 

757 getKeysAsTuples(self._data, keys, None) 

758 return keys 

759 

760 def names(self, topLevelOnly=False, delimiter=None): 

761 """Get a delimited name of all the keys in the hierarchy. 

762 

763 The values returned from this method are guaranteed to be usable 

764 to access items in the configuration object. 

765 

766 Parameters 

767 ---------- 

768 topLevelOnly : `bool`, optional 

769 If False, the default, a full hierarchy of names is returned. 

770 If True, only the top level are returned. 

771 delimiter : `str`, optional 

772 Delimiter to use when forming the keys. If the delimiter is 

773 present in any of the keys, it will be escaped in the returned 

774 names. If `None` given a delimiter will be automatically provided. 

775 The delimiter can not be alphanumeric. 

776 

777 Returns 

778 ------- 

779 names : `list` of `str` 

780 List of all names present in the `Config`. 

781 

782 Notes 

783 ----- 

784 This is different than the built-in method `dict.keys`, which will 

785 return only the first level keys. 

786 

787 Raises 

788 ------ 

789 ValueError: 

790 The supplied delimiter is alphanumeric. 

791 """ 

792 if topLevelOnly: 

793 return list(self.keys()) 

794 

795 # Get all the tuples of hierarchical keys 

796 nameTuples = self.nameTuples() 

797 

798 if delimiter is not None and delimiter.isalnum(): 

799 raise ValueError(f"Supplied delimiter ({delimiter!r}) must not be alphanumeric.") 

800 

801 if delimiter is None: 

802 # Start with something, and ensure it does not need to be 

803 # escaped (it is much easier to understand if not escaped) 

804 delimiter = self._D 

805 

806 # Form big string for easy check of delimiter clash 

807 combined = "".join("".join(str(s) for s in k) for k in nameTuples) 

808 

809 # Try a delimiter and keep trying until we get something that 

810 # works. 

811 ntries = 0 

812 while delimiter in combined: 

813 log.debug("Delimiter '%s' could not be used. Trying another.", delimiter) 

814 ntries += 1 

815 

816 if ntries > 100: 

817 raise ValueError(f"Unable to determine a delimiter for Config {self}") 

818 

819 # try another one 

820 while True: 

821 delimiter = chr(ord(delimiter) + 1) 

822 if not delimiter.isalnum(): 

823 break 

824 

825 log.debug("Using delimiter %r", delimiter) 

826 

827 # Form the keys, escaping the delimiter if necessary 

828 strings = [ 

829 delimiter + delimiter.join(str(s).replace(delimiter, f"\\{delimiter}") for s in k) 

830 for k in nameTuples 

831 ] 

832 return strings 

833 

834 def asArray(self, name): 

835 """Get a value as an array. 

836 

837 May contain one or more elements. 

838 

839 Parameters 

840 ---------- 

841 name : `str` 

842 Key to use to retrieve value. 

843 

844 Returns 

845 ------- 

846 array : `collections.abc.Sequence` 

847 The value corresponding to name, but guaranteed to be returned 

848 as a list with at least one element. If the value is a 

849 `~collections.abc.Sequence` (and not a `str`) the value itself 

850 will be returned, else the value will be the first element. 

851 """ 

852 val = self.get(name) 

853 if isinstance(val, str): 

854 val = [val] 

855 elif not isinstance(val, collections.abc.Sequence): 

856 val = [val] 

857 return val 

858 

859 def __eq__(self, other): 

860 if isinstance(other, Config): 

861 other = other._data 

862 return self._data == other 

863 

864 def __ne__(self, other): 

865 if isinstance(other, Config): 

866 other = other._data 

867 return self._data != other 

868 

869 ####### 

870 # i/o # 

871 

872 def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]: 

873 """Write the config to an output stream. 

874 

875 Parameters 

876 ---------- 

877 output : `IO`, optional 

878 The stream to use for output. If `None` the serialized content 

879 will be returned. 

880 format : `str`, optional 

881 The format to use for the output. Can be "yaml" or "json". 

882 

883 Returns 

884 ------- 

885 serialized : `str` or `None` 

886 If a stream was given the stream will be used and the return 

887 value will be `None`. If the stream was `None` the 

888 serialization will be returned as a string. 

889 """ 

890 if format == "yaml": 

891 return yaml.safe_dump(self._data, output, default_flow_style=False) 

892 elif format == "json": 

893 if output is not None: 

894 json.dump(self._data, output, ensure_ascii=False) 

895 return None 

896 else: 

897 return json.dumps(self._data, ensure_ascii=False) 

898 raise ValueError(f"Unsupported format for Config serialization: {format}") 

899 

900 def dumpToUri( 

901 self, 

902 uri: ResourcePathExpression, 

903 updateFile: bool = True, 

904 defaultFileName: str = "butler.yaml", 

905 overwrite: bool = True, 

906 ) -> None: 

907 """Write the config to location pointed to by given URI. 

908 

909 Currently supports 's3' and 'file' URI schemes. 

910 

911 Parameters 

912 ---------- 

913 uri: `lsst.resources.ResourcePathExpression` 

914 URI of location where the Config will be written. 

915 updateFile : bool, optional 

916 If True and uri does not end on a filename with extension, will 

917 append `defaultFileName` to the target uri. True by default. 

918 defaultFileName : bool, optional 

919 The file name that will be appended to target uri if updateFile is 

920 True and uri does not end on a file with an extension. 

921 overwrite : bool, optional 

922 If True the configuration will be written even if it already 

923 exists at that location. 

924 """ 

925 # Make local copy of URI or create new one 

926 uri = ResourcePath(uri) 

927 

928 if updateFile and not uri.getExtension(): 

929 uri = uri.updatedFile(defaultFileName) 

930 

931 # Try to work out the format from the extension 

932 ext = uri.getExtension() 

933 format = ext[1:].lower() 

934 

935 output = self.dump(format=format) 

936 assert output is not None, "Config.dump guarantees not-None return when output arg is None" 

937 uri.write(output.encode(), overwrite=overwrite) 

938 self.configFile = uri 

939 

940 @staticmethod 

941 def updateParameters(configType, config, full, toUpdate=None, toCopy=None, overwrite=True, toMerge=None): 

942 """Update specific config parameters. 

943 

944 Allows for named parameters to be set to new values in bulk, and 

945 for other values to be set by copying from a reference config. 

946 

947 Assumes that the supplied config is compatible with ``configType`` 

948 and will attach the updated values to the supplied config by 

949 looking for the related component key. It is assumed that 

950 ``config`` and ``full`` are from the same part of the 

951 configuration hierarchy. 

952 

953 Parameters 

954 ---------- 

955 configType : `ConfigSubset` 

956 Config type to use to extract relevant items from ``config``. 

957 config : `Config` 

958 A `Config` to update. Only the subset understood by 

959 the supplied `ConfigSubset` will be modified. Default values 

960 will not be inserted and the content will not be validated 

961 since mandatory keys are allowed to be missing until 

962 populated later by merging. 

963 full : `Config` 

964 A complete config with all defaults expanded that can be 

965 converted to a ``configType``. Read-only and will not be 

966 modified by this method. Values are read from here if 

967 ``toCopy`` is defined. 

968 

969 Repository-specific options that should not be obtained 

970 from defaults when Butler instances are constructed 

971 should be copied from ``full`` to ``config``. 

972 toUpdate : `dict`, optional 

973 A `dict` defining the keys to update and the new value to use. 

974 The keys and values can be any supported by `Config` 

975 assignment. 

976 toCopy : `tuple`, optional 

977 `tuple` of keys whose values should be copied from ``full`` 

978 into ``config``. 

979 overwrite : `bool`, optional 

980 If `False`, do not modify a value in ``config`` if the key 

981 already exists. Default is always to overwrite. 

982 toMerge : `tuple`, optional 

983 Keys to merge content from full to config without overwriting 

984 pre-existing values. Only works if the key refers to a hierarchy. 

985 The ``overwrite`` flag is ignored. 

986 

987 Raises 

988 ------ 

989 ValueError 

990 Neither ``toUpdate``, ``toCopy`` nor ``toMerge`` were defined. 

991 """ 

992 if toUpdate is None and toCopy is None and toMerge is None: 

993 raise ValueError("At least one of toUpdate, toCopy, or toMerge parameters must be set.") 

994 

995 # If this is a parent configuration then we need to ensure that 

996 # the supplied config has the relevant component key in it. 

997 # If this is a parent configuration we add in the stub entry 

998 # so that the ConfigSubset constructor will do the right thing. 

999 # We check full for this since that is guaranteed to be complete. 

1000 if configType.component in full and configType.component not in config: 

1001 config[configType.component] = {} 

1002 

1003 # Extract the part of the config we wish to update 

1004 localConfig = configType(config, mergeDefaults=False, validate=False) 

1005 

1006 if toUpdate: 

1007 for key, value in toUpdate.items(): 

1008 if key in localConfig and not overwrite: 

1009 log.debug( 

1010 "Not overriding key '%s' with value '%s' in config %s", 

1011 key, 

1012 value, 

1013 localConfig.__class__.__name__, 

1014 ) 

1015 else: 

1016 localConfig[key] = value 

1017 

1018 if toCopy or toMerge: 

1019 localFullConfig = configType(full, mergeDefaults=False) 

1020 

1021 if toCopy: 

1022 for key in toCopy: 

1023 if key in localConfig and not overwrite: 

1024 log.debug( 

1025 "Not overriding key '%s' from defaults in config %s", 

1026 key, 

1027 localConfig.__class__.__name__, 

1028 ) 

1029 else: 

1030 localConfig[key] = localFullConfig[key] 

1031 if toMerge: 

1032 for key in toMerge: 

1033 if key in localConfig: 

1034 # Get the node from the config to do the merge 

1035 # but then have to reattach to the config. 

1036 subset = localConfig[key] 

1037 subset.merge(localFullConfig[key]) 

1038 localConfig[key] = subset 

1039 else: 

1040 localConfig[key] = localFullConfig[key] 

1041 

1042 # Reattach to parent if this is a child config 

1043 if configType.component in config: 

1044 config[configType.component] = localConfig 

1045 else: 

1046 config.update(localConfig) 

1047 

1048 def toDict(self): 

1049 """Convert a `Config` to a standalone hierarchical `dict`. 

1050 

1051 Returns 

1052 ------- 

1053 d : `dict` 

1054 The standalone hierarchical `dict` with any `Config` classes 

1055 in the hierarchy converted to `dict`. 

1056 

1057 Notes 

1058 ----- 

1059 This can be useful when passing a Config to some code that 

1060 expects native Python types. 

1061 """ 

1062 output = copy.deepcopy(self._data) 

1063 for k, v in output.items(): 

1064 if isinstance(v, Config): 1064 ↛ 1065line 1064 didn't jump to line 1065, because the condition on line 1064 was never true

1065 v = v.toDict() 

1066 output[k] = v 

1067 return output 

1068 

1069 

1070class ConfigSubset(Config): 

1071 """Config representing a subset of a more general configuration. 

1072 

1073 Subclasses define their own component and when given a configuration 

1074 that includes that component, the resulting configuration only includes 

1075 the subset. For example, your config might contain ``dimensions`` if it's 

1076 part of a global config and that subset will be stored. If ``dimensions`` 

1077 can not be found it is assumed that the entire contents of the 

1078 configuration should be used. 

1079 

1080 Default values are read from the environment or supplied search paths 

1081 using the default configuration file name specified in the subclass. 

1082 This allows a configuration class to be instantiated without any 

1083 additional arguments. 

1084 

1085 Additional validation can be specified to check for keys that are mandatory 

1086 in the configuration. 

1087 

1088 Parameters 

1089 ---------- 

1090 other : `Config` or `str` or `dict` 

1091 Argument specifying the configuration information as understood 

1092 by `Config` 

1093 validate : `bool`, optional 

1094 If `True` required keys will be checked to ensure configuration 

1095 consistency. 

1096 mergeDefaults : `bool`, optional 

1097 If `True` defaults will be read and the supplied config will 

1098 be combined with the defaults, with the supplied values taking 

1099 precedence. 

1100 searchPaths : `list` or `tuple`, optional 

1101 Explicit additional paths to search for defaults. They should 

1102 be supplied in priority order. These paths have higher priority 

1103 than those read from the environment in 

1104 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

1105 the local file system or URIs, `lsst.resources.ResourcePath`. 

1106 """ 

1107 

1108 component: ClassVar[Optional[str]] = None 

1109 """Component to use from supplied config. Can be None. If specified the 

1110 key is not required. Can be a full dot-separated path to a component. 

1111 """ 

1112 

1113 requiredKeys: ClassVar[Sequence[str]] = () 

1114 """Keys that are required to be specified in the configuration. 

1115 """ 

1116 

1117 defaultConfigFile: ClassVar[Optional[str]] = None 

1118 """Name of the file containing defaults for this config class. 

1119 """ 

1120 

1121 def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None): 

1122 # Create a blank object to receive the defaults 

1123 # Once we have the defaults we then update with the external values 

1124 super().__init__() 

1125 

1126 # Create a standard Config rather than subset 

1127 externalConfig = Config(other) 

1128 

1129 # Select the part we need from it 

1130 # To simplify the use of !include we also check for the existence of 

1131 # component.component (since the included files can themselves 

1132 # include the component name) 

1133 if self.component is not None: 1133 ↛ 1142line 1133 didn't jump to line 1142, because the condition on line 1133 was never false

1134 doubled = (self.component, self.component) 

1135 # Must check for double depth first 

1136 if doubled in externalConfig: 1136 ↛ 1137line 1136 didn't jump to line 1137, because the condition on line 1136 was never true

1137 externalConfig = externalConfig[doubled] 

1138 elif self.component in externalConfig: 

1139 externalConfig._data = externalConfig._data[self.component] 

1140 

1141 # Default files read to create this configuration 

1142 self.filesRead = [] 

1143 

1144 # Assume we are not looking up child configurations 

1145 containerKey = None 

1146 

1147 # Sometimes we do not want to merge with defaults. 

1148 if mergeDefaults: 

1149 # Supplied search paths have highest priority 

1150 fullSearchPath = [] 

1151 if searchPaths: 1151 ↛ 1152line 1151 didn't jump to line 1152, because the condition on line 1151 was never true

1152 fullSearchPath.extend(searchPaths) 

1153 

1154 # Read default paths from environment 

1155 fullSearchPath.extend(self.defaultSearchPaths()) 

1156 

1157 # There are two places to find defaults for this particular config 

1158 # - The "defaultConfigFile" defined in the subclass 

1159 # - The class specified in the "cls" element in the config. 

1160 # Read cls after merging in case it changes. 

1161 if self.defaultConfigFile is not None: 1161 ↛ 1166line 1161 didn't jump to line 1166, because the condition on line 1161 was never false

1162 self._updateWithConfigsFromPath(fullSearchPath, self.defaultConfigFile) 

1163 

1164 # Can have a class specification in the external config (priority) 

1165 # or from the defaults. 

1166 pytype = None 

1167 if "cls" in externalConfig: 1167 ↛ 1168line 1167 didn't jump to line 1168, because the condition on line 1167 was never true

1168 pytype = externalConfig["cls"] 

1169 elif "cls" in self: 1169 ↛ 1170line 1169 didn't jump to line 1170, because the condition on line 1169 was never true

1170 pytype = self["cls"] 

1171 

1172 if pytype is not None: 1172 ↛ 1173line 1172 didn't jump to line 1173, because the condition on line 1172 was never true

1173 try: 

1174 cls = doImport(pytype) 

1175 except ImportError as e: 

1176 raise RuntimeError(f"Failed to import cls '{pytype}' for config {type(self)}") from e 

1177 defaultsFile = cls.defaultConfigFile 

1178 if defaultsFile is not None: 

1179 self._updateWithConfigsFromPath(fullSearchPath, defaultsFile) 

1180 

1181 # Get the container key in case we need it 

1182 try: 

1183 containerKey = cls.containerKey 

1184 except AttributeError: 

1185 pass 

1186 

1187 # Now update this object with the external values so that the external 

1188 # values always override the defaults 

1189 self.update(externalConfig) 

1190 if not self.configFile: 1190 ↛ 1196line 1190 didn't jump to line 1196, because the condition on line 1190 was never false

1191 self.configFile = externalConfig.configFile 

1192 

1193 # If this configuration has child configurations of the same 

1194 # config class, we need to expand those defaults as well. 

1195 

1196 if mergeDefaults and containerKey is not None and containerKey in self: 1196 ↛ 1197line 1196 didn't jump to line 1197, because the condition on line 1196 was never true

1197 for idx, subConfig in enumerate(self[containerKey]): 

1198 self[containerKey, idx] = type(self)( 

1199 other=subConfig, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths 

1200 ) 

1201 

1202 if validate: 

1203 self.validate() 

1204 

1205 @classmethod 

1206 def defaultSearchPaths(cls): 

1207 """Read environment to determine search paths to use. 

1208 

1209 Global defaults, at lowest priority, are found in the ``config`` 

1210 directory of the butler source tree. Additional defaults can be 

1211 defined using the environment variable ``$DAF_BUTLER_CONFIG_PATHS`` 

1212 which is a PATH-like variable where paths at the front of the list 

1213 have priority over those later. 

1214 

1215 Returns 

1216 ------- 

1217 paths : `list` 

1218 Returns a list of paths to search. The returned order is in 

1219 priority with the highest priority paths first. The butler config 

1220 configuration resources will not be included here but will 

1221 always be searched last. 

1222 

1223 Notes 

1224 ----- 

1225 The environment variable is split on the standard ``:`` path separator. 

1226 This currently makes it incompatible with usage of URIs. 

1227 """ 

1228 # We can pick up defaults from multiple search paths 

1229 # We fill defaults by using the butler config path and then 

1230 # the config path environment variable in reverse order. 

1231 defaultsPaths: List[Union[str, ResourcePath]] = [] 

1232 

1233 if CONFIG_PATH in os.environ: 1233 ↛ 1234line 1233 didn't jump to line 1234, because the condition on line 1233 was never true

1234 externalPaths = os.environ[CONFIG_PATH].split(os.pathsep) 

1235 defaultsPaths.extend(externalPaths) 

1236 

1237 # Add the package defaults as a resource 

1238 defaultsPaths.append(ResourcePath(f"resource://{cls.resourcesPackage}/configs", forceDirectory=True)) 

1239 return defaultsPaths 

1240 

1241 def _updateWithConfigsFromPath(self, searchPaths, configFile): 

1242 """Search the supplied paths, merging the configuration values. 

1243 

1244 The values read will override values currently stored in the object. 

1245 Every file found in the path will be read, such that the earlier 

1246 path entries have higher priority. 

1247 

1248 Parameters 

1249 ---------- 

1250 searchPaths : `list` of `lsst.resources.ResourcePath`, `str` 

1251 Paths to search for the supplied configFile. This path 

1252 is the priority order, such that files read from the 

1253 first path entry will be selected over those read from 

1254 a later path. Can contain `str` referring to the local file 

1255 system or a URI string. 

1256 configFile : `lsst.resources.ResourcePath` 

1257 File to locate in path. If absolute path it will be read 

1258 directly and the search path will not be used. Can be a URI 

1259 to an explicit resource (which will ignore the search path) 

1260 which is assumed to exist. 

1261 """ 

1262 uri = ResourcePath(configFile) 

1263 if uri.isabs() and uri.exists(): 1263 ↛ 1265line 1263 didn't jump to line 1265, because the condition on line 1263 was never true

1264 # Assume this resource exists 

1265 self._updateWithOtherConfigFile(configFile) 

1266 self.filesRead.append(configFile) 

1267 else: 

1268 # Reverse order so that high priority entries 

1269 # update the object last. 

1270 for pathDir in reversed(searchPaths): 

1271 if isinstance(pathDir, (str, ResourcePath)): 1271 ↛ 1278line 1271 didn't jump to line 1278, because the condition on line 1271 was never false

1272 pathDir = ResourcePath(pathDir, forceDirectory=True) 

1273 file = pathDir.join(configFile) 

1274 if file.exists(): 1274 ↛ 1270line 1274 didn't jump to line 1270, because the condition on line 1274 was never false

1275 self.filesRead.append(file) 

1276 self._updateWithOtherConfigFile(file) 

1277 else: 

1278 raise ValueError(f"Unexpected search path type encountered: {pathDir!r}") 

1279 

1280 def _updateWithOtherConfigFile(self, file): 

1281 """Read in some defaults and update. 

1282 

1283 Update the configuration by reading the supplied file as a config 

1284 of this class, and merging such that these values override the 

1285 current values. Contents of the external config are not validated. 

1286 

1287 Parameters 

1288 ---------- 

1289 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

1290 Entity that can be converted to a `ConfigSubset`. 

1291 """ 

1292 # Use this class to read the defaults so that subsetting can happen 

1293 # correctly. 

1294 externalConfig = type(self)(file, validate=False, mergeDefaults=False) 

1295 self.update(externalConfig) 

1296 

1297 def validate(self): 

1298 """Check that mandatory keys are present in this configuration. 

1299 

1300 Ignored if ``requiredKeys`` is empty. 

1301 """ 

1302 # Validation 

1303 missing = [k for k in self.requiredKeys if k not in self._data] 

1304 if missing: 1304 ↛ 1305line 1304 didn't jump to line 1305, because the condition on line 1304 was never true

1305 raise KeyError(f"Mandatory keys ({missing}) missing from supplied configuration for {type(self)}")