Coverage for python/lsst/pex/config/config.py: 58%

458 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-14 02:34 -0800

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from typing import Any, ForwardRef, Generic, Mapping, Optional, TypeVar, Union, cast, overload 

49 

50try: 

51 from types import GenericAlias 

52except ImportError: 

53 # cover python 3.8 usage 

54 GenericAlias = type(Mapping[int, int]) 

55 

56# if YAML is not available that's fine and we simply don't register 

57# the yaml representer since we know it won't be used. 

58try: 

59 import yaml 

60except ImportError: 

61 yaml = None 

62 

63from .callStack import getCallStack, getStackFrame 

64from .comparison import compareConfigs, compareScalars, getComparisonName 

65 

66if yaml: 66 ↛ 77line 66 didn't jump to line 77, because the condition on line 66 was never false

67 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

68 

69 try: 

70 # CLoader is not always available 

71 from yaml import CLoader 

72 

73 YamlLoaders += (CLoader,) 

74 except ImportError: 

75 pass 

76else: 

77 YamlLoaders = () 

78 doImport = None 

79 

80 

81if int(sys.version_info.minor) < 9: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true

82 genericAliasKwds = {"_root": True} 

83else: 

84 genericAliasKwds = {} 

85 

86 

87class _PexConfigGenericAlias(GenericAlias, **genericAliasKwds): 

88 """A Subclass of python's GenericAlias used in defining and instantiating 

89 Generics. 

90 

91 This class differs from `types.GenericAlias` in that it calls a method 

92 named _parseTypingArgs defined on Fields. This method gives Field and its 

93 subclasses an opportunity to transform type parameters into class key word 

94 arguments. Code authors do not need to implement any returns of this object 

95 directly, and instead only need implement _parseTypingArgs, if a Field 

96 subclass differs from the base class implementation. 

97 

98 This class is intended to be an implementation detail, returned from a 

99 Field's `__class_getitem__` method. 

100 """ 

101 

102 def __call__(self, *args: Any, **kwds: Any) -> Any: 

103 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

104 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

105 

106 

107FieldTypeVar = TypeVar("FieldTypeVar") 

108 

109 

110class UnexpectedProxyUsageError(TypeError): 

111 """Exception raised when a proxy class is used in a context that suggests 

112 it should have already been converted to the thing it proxies. 

113 """ 

114 

115 

116def _joinNamePath(prefix=None, name=None, index=None): 

117 """Generate nested configuration names.""" 

118 if not prefix and not name: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 raise ValueError("Invalid name: cannot be None") 

120 elif not name: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 name = prefix 

122 elif prefix and name: 122 ↛ 125line 122 didn't jump to line 125, because the condition on line 122 was never false

123 name = prefix + "." + name 

124 

125 if index is not None: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true

126 return "%s[%r]" % (name, index) 

127 else: 

128 return name 

129 

130 

131def _autocast(x, dtype): 

132 """Cast a value to a type, if appropriate. 

133 

134 Parameters 

135 ---------- 

136 x : object 

137 A value. 

138 dtype : tpye 

139 Data type, such as `float`, `int`, or `str`. 

140 

141 Returns 

142 ------- 

143 values : object 

144 If appropriate, the returned value is ``x`` cast to the given type 

145 ``dtype``. If the cast cannot be performed the original value of 

146 ``x`` is returned. 

147 """ 

148 if dtype == float and isinstance(x, int): 

149 return float(x) 

150 return x 

151 

152 

153def _typeStr(x): 

154 """Generate a fully-qualified type name. 

155 

156 Returns 

157 ------- 

158 `str` 

159 Fully-qualified type name. 

160 

161 Notes 

162 ----- 

163 This function is used primarily for writing config files to be executed 

164 later upon with the 'load' function. 

165 """ 

166 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

167 xtype = x 

168 else: 

169 xtype = type(x) 

170 if (sys.version_info.major <= 2 and xtype.__module__ == "__builtin__") or xtype.__module__ == "builtins": 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 return xtype.__name__ 

172 else: 

173 return "%s.%s" % (xtype.__module__, xtype.__name__) 

174 

175 

176if yaml: 176 ↛ 209line 176 didn't jump to line 209, because the condition on line 176 was never false

177 

178 def _yaml_config_representer(dumper, data): 

179 """Represent a Config object in a form suitable for YAML. 

180 

181 Stores the serialized stream as a scalar block string. 

182 """ 

183 stream = io.StringIO() 

184 data.saveToStream(stream) 

185 config_py = stream.getvalue() 

186 

187 # Strip multiple newlines from the end of the config 

188 # This simplifies the YAML to use | and not |+ 

189 config_py = config_py.rstrip() + "\n" 

190 

191 # Trailing spaces force pyyaml to use non-block form. 

192 # Remove the trailing spaces so it has no choice 

193 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

194 

195 # Store the Python as a simple scalar 

196 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

197 

198 def _yaml_config_constructor(loader, node): 

199 """Construct a config from YAML""" 

200 config_py = loader.construct_scalar(node) 

201 return Config._fromPython(config_py) 

202 

203 # Register a generic constructor for Config and all subclasses 

204 # Need to register for all the loaders we would like to use 

205 for loader in YamlLoaders: 

206 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

207 

208 

209class ConfigMeta(type): 

210 """A metaclass for `lsst.pex.config.Config`. 

211 

212 Notes 

213 ----- 

214 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

215 class attributes as a class attribute called ``_fields``, and adds 

216 the name of each field as an instance variable of the field itself (so you 

217 don't have to pass the name of the field to the field constructor). 

218 """ 

219 

220 def __init__(cls, name, bases, dict_): 

221 type.__init__(cls, name, bases, dict_) 

222 cls._fields = {} 

223 cls._source = getStackFrame() 

224 

225 def getFields(classtype): 

226 fields = {} 

227 bases = list(classtype.__bases__) 

228 bases.reverse() 

229 for b in bases: 

230 fields.update(getFields(b)) 

231 

232 for k, v in classtype.__dict__.items(): 

233 if isinstance(v, Field): 

234 fields[k] = v 

235 return fields 

236 

237 fields = getFields(cls) 

238 for k, v in fields.items(): 

239 setattr(cls, k, copy.deepcopy(v)) 

240 

241 def __setattr__(cls, name, value): 

242 if isinstance(value, Field): 

243 value.name = name 

244 cls._fields[name] = value 

245 type.__setattr__(cls, name, value) 

246 

247 

248class FieldValidationError(ValueError): 

249 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

250 particular ``~lsst.pex.config.Config``. 

251 

252 Parameters 

253 ---------- 

254 field : `lsst.pex.config.Field` 

255 The field that was not valid. 

256 config : `lsst.pex.config.Config` 

257 The config containing the invalid field. 

258 msg : `str` 

259 Text describing why the field was not valid. 

260 """ 

261 

262 def __init__(self, field, config, msg): 

263 self.fieldType = type(field) 

264 """Type of the `~lsst.pex.config.Field` that incurred the error. 

265 """ 

266 

267 self.fieldName = field.name 

268 """Name of the `~lsst.pex.config.Field` instance that incurred the 

269 error (`str`). 

270 

271 See also 

272 -------- 

273 lsst.pex.config.Field.name 

274 """ 

275 

276 self.fullname = _joinNamePath(config._name, field.name) 

277 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

278 (`str`). 

279 """ 

280 

281 self.history = config.history.setdefault(field.name, []) 

282 """Full history of all changes to the `~lsst.pex.config.Field` 

283 instance. 

284 """ 

285 

286 self.fieldSource = field.source 

287 """File and line number of the `~lsst.pex.config.Field` definition. 

288 """ 

289 

290 self.configSource = config._source 

291 error = ( 

292 "%s '%s' failed validation: %s\n" 

293 "For more information see the Field definition at:\n%s" 

294 " and the Config definition at:\n%s" 

295 % ( 

296 self.fieldType.__name__, 

297 self.fullname, 

298 msg, 

299 self.fieldSource.format(), 

300 self.configSource.format(), 

301 ) 

302 ) 

303 super().__init__(error) 

304 

305 

306class Field(Generic[FieldTypeVar]): 

307 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

308 `complex`, `bool`, and `str` data types. 

309 

310 Parameters 

311 ---------- 

312 doc : `str` 

313 A description of the field for users. 

314 dtype : type, optional 

315 The field's data type. ``Field`` only supports basic data types: 

316 `int`, `float`, `complex`, `bool`, and `str`. See 

317 `Field.supportedTypes`. Optional if supplied as a typing argument to 

318 the class. 

319 default : object, optional 

320 The field's default value. 

321 check : callable, optional 

322 A callable that is called with the field's value. This callable should 

323 return `False` if the value is invalid. More complex inter-field 

324 validation can be written as part of the 

325 `lsst.pex.config.Config.validate` method. 

326 optional : `bool`, optional 

327 This sets whether the field is considered optional, and therefore 

328 doesn't need to be set by the user. When `False`, 

329 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

330 deprecated : None or `str`, optional 

331 A description of why this Field is deprecated, including removal date. 

332 If not None, the string is appended to the docstring for this Field. 

333 

334 Raises 

335 ------ 

336 ValueError 

337 Raised when the ``dtype`` parameter is not one of the supported types 

338 (see `Field.supportedTypes`). 

339 

340 See also 

341 -------- 

342 ChoiceField 

343 ConfigChoiceField 

344 ConfigDictField 

345 ConfigField 

346 ConfigurableField 

347 DictField 

348 ListField 

349 RangeField 

350 RegistryField 

351 

352 Notes 

353 ----- 

354 ``Field`` instances (including those of any subclass of ``Field``) are used 

355 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

356 example, below). ``Field`` attributes work like the `property` attributes 

357 of classes that implement custom setters and getters. `Field` attributes 

358 belong to the class, but operate on the instance. Formally speaking, 

359 `Field` attributes are `descriptors 

360 <https://docs.python.org/3/howto/descriptor.html>`_. 

361 

362 When you access a `Field` attribute on a `Config` instance, you don't 

363 get the `Field` instance itself. Instead, you get the value of that field, 

364 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

365 container type (like a `lsst.pex.config.List`) depending on the field's 

366 type. See the example, below. 

367 

368 Fields can be annotated with a type similar to other python classes (python 

369 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

370 See the name field in the Config example below for an example of this. 

371 Unlike most other uses in python, this has an effect at type checking *and* 

372 runtime. If the type is specified with a class annotation, it will be used 

373 as the value of the ``dtype`` in the ``Field`` and there is no need to 

374 specify it as an argument during instantiation. 

375 

376 There are Some notes on dtype through type annotation syntax. Type 

377 annotation syntax supports supplying the argument as a string of a type 

378 name. i.e. "float", but this cannot be used to resolve circular references. 

379 Type annotation syntax can be used on an identifier in addition to Class 

380 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

381 ``someField = Field[str](doc="some doc"). However, this syntax is only 

382 useful for annotating the type of the identifier (i.e. variable in previous 

383 example) and does nothing for assigning the dtype of the ``Field``. 

384 

385 

386 Examples 

387 -------- 

388 Instances of ``Field`` should be used as class attributes of 

389 `lsst.pex.config.Config` subclasses: 

390 

391 >>> from lsst.pex.config import Config, Field 

392 >>> class Example(Config): 

393 ... myInt = Field("An integer field.", int, default=0) 

394 ... name = Field[str](doc="A string Field") 

395 ... 

396 >>> print(config.myInt) 

397 0 

398 >>> config.myInt = 5 

399 >>> print(config.myInt) 

400 5 

401 """ 

402 

403 name: str 

404 """Identifier (variable name) used to refer to a Field within a Config 

405 Class. 

406 """ 

407 

408 supportedTypes = set((str, bool, float, int, complex)) 

409 """Supported data types for field values (`set` of types). 

410 """ 

411 

412 @staticmethod 

413 def _parseTypingArgs( 

414 params: Union[tuple[type, ...], tuple[str, ...]], kwds: Mapping[str, Any] 

415 ) -> Mapping[str, Any]: 

416 """Parses type annotations into keyword constructor arguments. 

417 

418 This is a special private method that interprets type arguments (i.e. 

419 Field[str]) into keyword arguments to be passed on to the constructor. 

420 

421 Subclasses of Field can implement this method to customize how they 

422 handle turning type parameters into keyword arguments (see DictField 

423 for an example) 

424 

425 Parameters 

426 ---------- 

427 params : `tuple` of `type` or `tuple` of str 

428 Parameters passed to the type annotation. These will either be 

429 types or strings. Strings are to interpreted as forward references 

430 and will be treated as such. 

431 kwds : `MutableMapping` with keys of `str` and values of `Any` 

432 These are the user supplied keywords that are to be passed to the 

433 Field constructor. 

434 

435 Returns 

436 ------- 

437 kwds : `MutableMapping` with keys of `str` and values of `Any` 

438 The mapping of keywords that will be passed onto the constructor 

439 of the Field. Should be filled in with any information gleaned 

440 from the input parameters. 

441 

442 Raises 

443 ------ 

444 ValueError : 

445 Raised if params is of incorrect length. 

446 Raised if a forward reference could not be resolved 

447 Raised if there is a conflict between params and values in kwds 

448 """ 

449 if len(params) > 1: 

450 raise ValueError("Only single type parameters are supported") 

451 unpackedParams = params[0] 

452 if isinstance(unpackedParams, str): 

453 _typ = ForwardRef(unpackedParams) 

454 # type ignore below because typeshed seems to be wrong. It 

455 # indicates there are only 2 args, as it was in python 3.8, but 

456 # 3.9+ takes 3 args. Attempt in old style and new style to 

457 # work with both. 

458 try: 

459 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

460 except TypeError: 

461 # python 3.8 path 

462 result = _typ._evaluate(globals(), locals()) 

463 if result is None: 

464 raise ValueError("Could not deduce type from input") 

465 unpackedParams = cast(type, result) 

466 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

467 raise ValueError("Conflicting definition for dtype") 

468 elif "dtype" not in kwds: 

469 kwds = {**kwds, **{"dtype": unpackedParams}} 

470 return kwds 

471 

472 def __class_getitem__(cls, params: Union[tuple[type, ...], type, ForwardRef]): 

473 return _PexConfigGenericAlias(cls, params) 

474 

475 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

476 if dtype is None: 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 raise ValueError( 

478 "dtype must either be supplied as an argument or as a type argument to the class" 

479 ) 

480 if dtype not in self.supportedTypes: 480 ↛ 481line 480 didn't jump to line 481, because the condition on line 480 was never true

481 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

482 

483 source = getStackFrame() 

484 self._setup( 

485 doc=doc, 

486 dtype=dtype, 

487 default=default, 

488 check=check, 

489 optional=optional, 

490 source=source, 

491 deprecated=deprecated, 

492 ) 

493 

494 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

495 """Set attributes, usually during initialization.""" 

496 self.dtype = dtype 

497 """Data type for the field. 

498 """ 

499 

500 # append the deprecation message to the docstring. 

501 if deprecated is not None: 

502 doc = f"{doc} Deprecated: {deprecated}" 

503 self.doc = doc 

504 """A description of the field (`str`). 

505 """ 

506 

507 self.deprecated = deprecated 

508 """If not None, a description of why this field is deprecated (`str`). 

509 """ 

510 

511 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

512 if optional or default is not None: 

513 self.__doc__ += f", default ``{default!r}``" 

514 self.__doc__ += ")" 

515 

516 self.default = default 

517 """Default value for this field. 

518 """ 

519 

520 self.check = check 

521 """A user-defined function that validates the value of the field. 

522 """ 

523 

524 self.optional = optional 

525 """Flag that determines if the field is required to be set (`bool`). 

526 

527 When `False`, `lsst.pex.config.Config.validate` will fail if the 

528 field's value is `None`. 

529 """ 

530 

531 self.source = source 

532 """The stack frame where this field is defined (`list` of 

533 `lsst.pex.config.callStack.StackFrame`). 

534 """ 

535 

536 def rename(self, instance): 

537 """Rename the field in a `~lsst.pex.config.Config` (for internal use 

538 only). 

539 

540 Parameters 

541 ---------- 

542 instance : `lsst.pex.config.Config` 

543 The config instance that contains this field. 

544 

545 Notes 

546 ----- 

547 This method is invoked by the `lsst.pex.config.Config` object that 

548 contains this field and should not be called directly. 

549 

550 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

551 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should 

552 rename each subconfig with the full field name as generated by 

553 `lsst.pex.config.config._joinNamePath`. 

554 """ 

555 pass 

556 

557 def validate(self, instance): 

558 """Validate the field (for internal use only). 

559 

560 Parameters 

561 ---------- 

562 instance : `lsst.pex.config.Config` 

563 The config instance that contains this field. 

564 

565 Raises 

566 ------ 

567 lsst.pex.config.FieldValidationError 

568 Raised if verification fails. 

569 

570 Notes 

571 ----- 

572 This method provides basic validation: 

573 

574 - Ensures that the value is not `None` if the field is not optional. 

575 - Ensures type correctness. 

576 - Ensures that the user-provided ``check`` function is valid. 

577 

578 Most `~lsst.pex.config.Field` subclasses should call 

579 `lsst.pex.config.field.Field.validate` if they re-implement 

580 `~lsst.pex.config.field.Field.validate`. 

581 """ 

582 value = self.__get__(instance) 

583 if not self.optional and value is None: 

584 raise FieldValidationError(self, instance, "Required value cannot be None") 

585 

586 def freeze(self, instance): 

587 """Make this field read-only (for internal use only). 

588 

589 Parameters 

590 ---------- 

591 instance : `lsst.pex.config.Config` 

592 The config instance that contains this field. 

593 

594 Notes 

595 ----- 

596 Freezing is only relevant for fields that hold subconfigs. Fields which 

597 hold subconfigs should freeze each subconfig. 

598 

599 **Subclasses should implement this method.** 

600 """ 

601 pass 

602 

603 def _validateValue(self, value): 

604 """Validate a value. 

605 

606 Parameters 

607 ---------- 

608 value : object 

609 The value being validated. 

610 

611 Raises 

612 ------ 

613 TypeError 

614 Raised if the value's type is incompatible with the field's 

615 ``dtype``. 

616 ValueError 

617 Raised if the value is rejected by the ``check`` method. 

618 """ 

619 if value is None: 619 ↛ 620line 619 didn't jump to line 620, because the condition on line 619 was never true

620 return 

621 

622 if not isinstance(value, self.dtype): 622 ↛ 623line 622 didn't jump to line 623, because the condition on line 622 was never true

623 msg = "Value %s is of incorrect type %s. Expected type %s" % ( 

624 value, 

625 _typeStr(value), 

626 _typeStr(self.dtype), 

627 ) 

628 raise TypeError(msg) 

629 if self.check is not None and not self.check(value): 629 ↛ 630line 629 didn't jump to line 630, because the condition on line 629 was never true

630 msg = "Value %s is not a valid value" % str(value) 

631 raise ValueError(msg) 

632 

633 def _collectImports(self, instance, imports): 

634 """This function should call the _collectImports method on all config 

635 objects the field may own, and union them with the supplied imports 

636 set. 

637 

638 Parameters 

639 ---------- 

640 instance : instance or subclass of `lsst.pex.config.Config` 

641 A config object that has this field defined on it 

642 imports : `set` 

643 Set of python modules that need imported after persistence 

644 """ 

645 pass 

646 

647 def save(self, outfile, instance): 

648 """Save this field to a file (for internal use only). 

649 

650 Parameters 

651 ---------- 

652 outfile : file-like object 

653 A writeable field handle. 

654 instance : `Config` 

655 The `Config` instance that contains this field. 

656 

657 Notes 

658 ----- 

659 This method is invoked by the `~lsst.pex.config.Config` object that 

660 contains this field and should not be called directly. 

661 

662 The output consists of the documentation string 

663 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

664 line is formatted as an assignment: ``{fullname}={value}``. 

665 

666 This output can be executed with Python. 

667 """ 

668 value = self.__get__(instance) 

669 fullname = _joinNamePath(instance._name, self.name) 

670 

671 if self.deprecated and value == self.default: 671 ↛ 672line 671 didn't jump to line 672, because the condition on line 671 was never true

672 return 

673 

674 # write full documentation string as comment lines 

675 # (i.e. first character is #) 

676 doc = "# " + str(self.doc).replace("\n", "\n# ") 

677 if isinstance(value, float) and not math.isfinite(value): 677 ↛ 679line 677 didn't jump to line 679, because the condition on line 677 was never true

678 # non-finite numbers need special care 

679 outfile.write("{}\n{}=float('{!r}')\n\n".format(doc, fullname, value)) 

680 else: 

681 outfile.write("{}\n{}={!r}\n\n".format(doc, fullname, value)) 

682 

683 def toDict(self, instance): 

684 """Convert the field value so that it can be set as the value of an 

685 item in a `dict` (for internal use only). 

686 

687 Parameters 

688 ---------- 

689 instance : `Config` 

690 The `Config` that contains this field. 

691 

692 Returns 

693 ------- 

694 value : object 

695 The field's value. See *Notes*. 

696 

697 Notes 

698 ----- 

699 This method invoked by the owning `~lsst.pex.config.Config` object and 

700 should not be called directly. 

701 

702 Simple values are passed through. Complex data structures must be 

703 manipulated. For example, a `~lsst.pex.config.Field` holding a 

704 subconfig should, instead of the subconfig object, return a `dict` 

705 where the keys are the field names in the subconfig, and the values are 

706 the field values in the subconfig. 

707 """ 

708 return self.__get__(instance) 

709 

710 @overload 

711 def __get__( 

712 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

713 ) -> "Field[FieldTypeVar]": 

714 ... 

715 

716 @overload 

717 def __get__( 

718 self, instance: "Config", owner: Any = None, at: Any = None, label: str = "default" 

719 ) -> FieldTypeVar: 

720 ... 

721 

722 def __get__(self, instance, owner=None, at=None, label="default"): 

723 """Define how attribute access should occur on the Config instance 

724 This is invoked by the owning config object and should not be called 

725 directly 

726 

727 When the field attribute is accessed on a Config class object, it 

728 returns the field object itself in order to allow inspection of 

729 Config classes. 

730 

731 When the field attribute is access on a config instance, the actual 

732 value described by the field (and held by the Config instance) is 

733 returned. 

734 """ 

735 if instance is None: 735 ↛ 736line 735 didn't jump to line 736, because the condition on line 735 was never true

736 return self 

737 else: 

738 # try statements are almost free in python if they succeed 

739 try: 

740 return instance._storage[self.name] 

741 except AttributeError: 

742 if not isinstance(instance, Config): 

743 return self 

744 else: 

745 raise AttributeError( 

746 f"Config {instance} is missing " 

747 "_storage attribute, likely" 

748 " incorrectly initialized" 

749 ) 

750 

751 def __set__( 

752 self, instance: "Config", value: Optional[FieldTypeVar], at: Any = None, label: str = "assignment" 

753 ) -> None: 

754 """Set an attribute on the config instance. 

755 

756 Parameters 

757 ---------- 

758 instance : `lsst.pex.config.Config` 

759 The config instance that contains this field. 

760 value : obj 

761 Value to set on this field. 

762 at : `list` of `lsst.pex.config.callStack.StackFrame` 

763 The call stack (created by 

764 `lsst.pex.config.callStack.getCallStack`). 

765 label : `str`, optional 

766 Event label for the history. 

767 

768 Notes 

769 ----- 

770 This method is invoked by the owning `lsst.pex.config.Config` object 

771 and should not be called directly. 

772 

773 Derived `~lsst.pex.config.Field` classes may need to override the 

774 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

775 should follow the following rules: 

776 

777 - Do not allow modification of frozen configs. 

778 - Validate the new value **before** modifying the field. Except if the 

779 new value is `None`. `None` is special and no attempt should be made 

780 to validate it until `lsst.pex.config.Config.validate` is called. 

781 - Do not modify the `~lsst.pex.config.Config` instance to contain 

782 invalid values. 

783 - If the field is modified, update the history of the 

784 `lsst.pex.config.field.Field` to reflect the changes. 

785 

786 In order to decrease the need to implement this method in derived 

787 `~lsst.pex.config.Field` types, value validation is performed in the 

788 `lsst.pex.config.Field._validateValue`. If only the validation step 

789 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

790 implement `lsst.pex.config.Field._validateValue` than to reimplement 

791 ``__set__``. More complicated behavior, however, may require 

792 reimplementation. 

793 """ 

794 if instance._frozen: 794 ↛ 795line 794 didn't jump to line 795, because the condition on line 794 was never true

795 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

796 

797 history = instance._history.setdefault(self.name, []) 

798 if value is not None: 798 ↛ 805line 798 didn't jump to line 805, because the condition on line 798 was never false

799 value = _autocast(value, self.dtype) 

800 try: 

801 self._validateValue(value) 

802 except BaseException as e: 

803 raise FieldValidationError(self, instance, str(e)) 

804 

805 instance._storage[self.name] = value 

806 if at is None: 806 ↛ 807line 806 didn't jump to line 807, because the condition on line 806 was never true

807 at = getCallStack() 

808 history.append((value, at, label)) 

809 

810 def __delete__(self, instance, at=None, label="deletion"): 

811 """Delete an attribute from a `lsst.pex.config.Config` instance. 

812 

813 Parameters 

814 ---------- 

815 instance : `lsst.pex.config.Config` 

816 The config instance that contains this field. 

817 at : `list` of `lsst.pex.config.callStack.StackFrame` 

818 The call stack (created by 

819 `lsst.pex.config.callStack.getCallStack`). 

820 label : `str`, optional 

821 Event label for the history. 

822 

823 Notes 

824 ----- 

825 This is invoked by the owning `~lsst.pex.config.Config` object and 

826 should not be called directly. 

827 """ 

828 if at is None: 

829 at = getCallStack() 

830 self.__set__(instance, None, at=at, label=label) 

831 

832 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

833 """Compare a field (named `Field.name`) in two 

834 `~lsst.pex.config.Config` instances for equality. 

835 

836 Parameters 

837 ---------- 

838 instance1 : `lsst.pex.config.Config` 

839 Left-hand side `Config` instance to compare. 

840 instance2 : `lsst.pex.config.Config` 

841 Right-hand side `Config` instance to compare. 

842 shortcut : `bool`, optional 

843 **Unused.** 

844 rtol : `float`, optional 

845 Relative tolerance for floating point comparisons. 

846 atol : `float`, optional 

847 Absolute tolerance for floating point comparisons. 

848 output : callable, optional 

849 A callable that takes a string, used (possibly repeatedly) to 

850 report inequalities. 

851 

852 Notes 

853 ----- 

854 This method must be overridden by more complex `Field` subclasses. 

855 

856 See also 

857 -------- 

858 lsst.pex.config.compareScalars 

859 """ 

860 v1 = getattr(instance1, self.name) 

861 v2 = getattr(instance2, self.name) 

862 name = getComparisonName( 

863 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

864 ) 

865 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

866 

867 

868class RecordingImporter: 

869 """Importer (for `sys.meta_path`) that records which modules are being 

870 imported. 

871 

872 *This class does not do any importing itself.* 

873 

874 Examples 

875 -------- 

876 Use this class as a context manager to ensure it is properly uninstalled 

877 when done: 

878 

879 >>> with RecordingImporter() as importer: 

880 ... # import stuff 

881 ... import numpy as np 

882 ... print("Imported: " + importer.getModules()) 

883 """ 

884 

885 def __init__(self): 

886 self._modules = set() 

887 

888 def __enter__(self): 

889 self.origMetaPath = sys.meta_path 

890 sys.meta_path = [self] + sys.meta_path # type: ignore 

891 return self 

892 

893 def __exit__(self, *args): 

894 self.uninstall() 

895 return False # Don't suppress exceptions 

896 

897 def uninstall(self): 

898 """Uninstall the importer.""" 

899 sys.meta_path = self.origMetaPath 

900 

901 def find_module(self, fullname, path=None): 

902 """Called as part of the ``import`` chain of events.""" 

903 self._modules.add(fullname) 

904 # Return None because we don't do any importing. 

905 return None 

906 

907 def getModules(self): 

908 """Get the set of modules that were imported. 

909 

910 Returns 

911 ------- 

912 modules : `set` of `str` 

913 Set of imported module names. 

914 """ 

915 return self._modules 

916 

917 

918# type ignore because type checker thinks ConfigMeta is Generic when it is not 

919class Config(metaclass=ConfigMeta): # type: ignore 

920 """Base class for configuration (*config*) objects. 

921 

922 Notes 

923 ----- 

924 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

925 instances as class attributes. These are used to define most of the base 

926 class behavior. 

927 

928 ``Config`` implements a mapping API that provides many `dict`-like methods, 

929 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and 

930 `itervalues`. ``Config`` instances also support the ``in`` operator to 

931 test if a field is in the config. Unlike a `dict`, ``Config`` classes are 

932 not subscriptable. Instead, access individual fields as attributes of the 

933 configuration instance. 

934 

935 Examples 

936 -------- 

937 Config classes are subclasses of ``Config`` that have 

938 `~lsst.pex.config.Field` instances (or instances of 

939 `~lsst.pex.config.Field` subclasses) as class attributes: 

940 

941 >>> from lsst.pex.config import Config, Field, ListField 

942 >>> class DemoConfig(Config): 

943 ... intField = Field(doc="An integer field", dtype=int, default=42) 

944 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

945 ... default=['coffee', 'green tea', 'water']) 

946 ... 

947 >>> config = DemoConfig() 

948 

949 Configs support many `dict`-like APIs: 

950 

951 >>> config.keys() 

952 ['intField', 'listField'] 

953 >>> 'intField' in config 

954 True 

955 

956 Individual fields can be accessed as attributes of the configuration: 

957 

958 >>> config.intField 

959 42 

960 >>> config.listField.append('earl grey tea') 

961 >>> print(config.listField) 

962 ['coffee', 'green tea', 'water', 'earl grey tea'] 

963 """ 

964 

965 _storage: dict[str, Any] 

966 _fields: dict[str, Field] 

967 _history: dict[str, list[Any]] 

968 _imports: set[Any] 

969 

970 def __iter__(self): 

971 """Iterate over fields.""" 

972 return self._fields.__iter__() 

973 

974 def keys(self): 

975 """Get field names. 

976 

977 Returns 

978 ------- 

979 names : `dict_keys` 

980 List of `lsst.pex.config.Field` names. 

981 

982 See also 

983 -------- 

984 lsst.pex.config.Config.iterkeys 

985 """ 

986 return self._storage.keys() 

987 

988 def values(self): 

989 """Get field values. 

990 

991 Returns 

992 ------- 

993 values : `dict_values` 

994 Iterator of field values. 

995 """ 

996 return self._storage.values() 

997 

998 def items(self): 

999 """Get configurations as ``(field name, field value)`` pairs. 

1000 

1001 Returns 

1002 ------- 

1003 items : `dict_items` 

1004 Iterator of tuples for each configuration. Tuple items are: 

1005 

1006 0. Field name. 

1007 1. Field value. 

1008 """ 

1009 return self._storage.items() 

1010 

1011 def __contains__(self, name): 

1012 """!Return True if the specified field exists in this config 

1013 

1014 @param[in] name field name to test for 

1015 """ 

1016 return self._storage.__contains__(name) 

1017 

1018 def __new__(cls, *args, **kw): 

1019 """Allocate a new `lsst.pex.config.Config` object. 

1020 

1021 In order to ensure that all Config object are always in a proper state 

1022 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1023 some attributes are handled at allocation time rather than at 

1024 initialization. 

1025 

1026 This ensures that even if a derived `~lsst.pex.config.Config` class 

1027 implements ``__init__``, its author does not need to be concerned about 

1028 when or even the base ``Config.__init__`` should be called. 

1029 """ 

1030 name = kw.pop("__name", None) 

1031 at = kw.pop("__at", getCallStack()) 

1032 # remove __label and ignore it 

1033 kw.pop("__label", "default") 

1034 

1035 instance = object.__new__(cls) 

1036 instance._frozen = False 

1037 instance._name = name 

1038 instance._storage = {} 

1039 instance._history = {} 

1040 instance._imports = set() 

1041 # load up defaults 

1042 for field in instance._fields.values(): 

1043 instance._history[field.name] = [] 

1044 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1045 # set custom default-overides 

1046 instance.setDefaults() 

1047 # set constructor overides 

1048 instance.update(__at=at, **kw) 

1049 return instance 

1050 

1051 def __reduce__(self): 

1052 """Reduction for pickling (function with arguments to reproduce). 

1053 

1054 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1055 since it may contain lambdas (as the ``check`` elements) that cannot 

1056 be pickled. 

1057 """ 

1058 # The stream must be in characters to match the API but pickle 

1059 # requires bytes 

1060 stream = io.StringIO() 

1061 self.saveToStream(stream) 

1062 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1063 

1064 def setDefaults(self): 

1065 """Subclass hook for computing defaults. 

1066 

1067 Notes 

1068 ----- 

1069 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1070 rather than using the `~lsst.pex.config.Field` instances's defaults 

1071 should do so here. To correctly use inherited defaults, 

1072 implementations of ``setDefaults`` must call their base class's 

1073 ``setDefaults``. 

1074 """ 

1075 pass 

1076 

1077 def update(self, **kw): 

1078 """Update values of fields specified by the keyword arguments. 

1079 

1080 Parameters 

1081 ---------- 

1082 kw 

1083 Keywords are configuration field names. Values are configuration 

1084 field values. 

1085 

1086 Notes 

1087 ----- 

1088 The ``__at`` and ``__label`` keyword arguments are special internal 

1089 keywords. They are used to strip out any internal steps from the 

1090 history tracebacks of the config. Do not modify these keywords to 

1091 subvert a `~lsst.pex.config.Config` instance's history. 

1092 

1093 Examples 

1094 -------- 

1095 This is a config with three fields: 

1096 

1097 >>> from lsst.pex.config import Config, Field 

1098 >>> class DemoConfig(Config): 

1099 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1100 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1101 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1102 ... 

1103 >>> config = DemoConfig() 

1104 

1105 These are the default values of each field: 

1106 

1107 >>> for name, value in config.iteritems(): 

1108 ... print(f"{name}: {value}") 

1109 ... 

1110 fieldA: 42 

1111 fieldB: True 

1112 fieldC: 'Hello world' 

1113 

1114 Using this method to update ``fieldA`` and ``fieldC``: 

1115 

1116 >>> config.update(fieldA=13, fieldC='Updated!') 

1117 

1118 Now the values of each field are: 

1119 

1120 >>> for name, value in config.iteritems(): 

1121 ... print(f"{name}: {value}") 

1122 ... 

1123 fieldA: 13 

1124 fieldB: True 

1125 fieldC: 'Updated!' 

1126 """ 

1127 at = kw.pop("__at", getCallStack()) 

1128 label = kw.pop("__label", "update") 

1129 

1130 for name, value in kw.items(): 

1131 try: 

1132 field = self._fields[name] 

1133 field.__set__(self, value, at=at, label=label) 

1134 except KeyError: 

1135 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) 

1136 

1137 def load(self, filename, root="config"): 

1138 """Modify this config in place by executing the Python code in a 

1139 configuration file. 

1140 

1141 Parameters 

1142 ---------- 

1143 filename : `str` 

1144 Name of the configuration file. A configuration file is Python 

1145 module. 

1146 root : `str`, optional 

1147 Name of the variable in file that refers to the config being 

1148 overridden. 

1149 

1150 For example, the value of root is ``"config"`` and the file 

1151 contains:: 

1152 

1153 config.myField = 5 

1154 

1155 Then this config's field ``myField`` is set to ``5``. 

1156 

1157 See also 

1158 -------- 

1159 lsst.pex.config.Config.loadFromStream 

1160 lsst.pex.config.Config.loadFromString 

1161 lsst.pex.config.Config.save 

1162 lsst.pex.config.Config.saveToStream 

1163 lsst.pex.config.Config.saveToString 

1164 """ 

1165 with open(filename, "r") as f: 

1166 code = compile(f.read(), filename=filename, mode="exec") 

1167 self.loadFromString(code, root=root, filename=filename) 

1168 

1169 def loadFromStream(self, stream, root="config", filename=None): 

1170 """Modify this Config in place by executing the Python code in the 

1171 provided stream. 

1172 

1173 Parameters 

1174 ---------- 

1175 stream : file-like object, `str`, `bytes`, or compiled string 

1176 Stream containing configuration override code. If this is a 

1177 code object, it should be compiled with ``mode="exec"``. 

1178 root : `str`, optional 

1179 Name of the variable in file that refers to the config being 

1180 overridden. 

1181 

1182 For example, the value of root is ``"config"`` and the file 

1183 contains:: 

1184 

1185 config.myField = 5 

1186 

1187 Then this config's field ``myField`` is set to ``5``. 

1188 filename : `str`, optional 

1189 Name of the configuration file, or `None` if unknown or contained 

1190 in the stream. Used for error reporting. 

1191 

1192 Notes 

1193 ----- 

1194 For backwards compatibility reasons, this method accepts strings, bytes 

1195 and code objects as well as file-like objects. New code should use 

1196 `loadFromString` instead for most of these types. 

1197 

1198 See also 

1199 -------- 

1200 lsst.pex.config.Config.load 

1201 lsst.pex.config.Config.loadFromString 

1202 lsst.pex.config.Config.save 

1203 lsst.pex.config.Config.saveToStream 

1204 lsst.pex.config.Config.saveToString 

1205 """ 

1206 if hasattr(stream, "read"): 1206 ↛ 1207line 1206 didn't jump to line 1207, because the condition on line 1206 was never true

1207 if filename is None: 

1208 filename = getattr(stream, "name", "?") 

1209 code = compile(stream.read(), filename=filename, mode="exec") 

1210 else: 

1211 code = stream 

1212 self.loadFromString(code, root=root, filename=filename) 

1213 

1214 def loadFromString(self, code, root="config", filename=None): 

1215 """Modify this Config in place by executing the Python code in the 

1216 provided string. 

1217 

1218 Parameters 

1219 ---------- 

1220 code : `str`, `bytes`, or compiled string 

1221 Stream containing configuration override code. 

1222 root : `str`, optional 

1223 Name of the variable in file that refers to the config being 

1224 overridden. 

1225 

1226 For example, the value of root is ``"config"`` and the file 

1227 contains:: 

1228 

1229 config.myField = 5 

1230 

1231 Then this config's field ``myField`` is set to ``5``. 

1232 filename : `str`, optional 

1233 Name of the configuration file, or `None` if unknown or contained 

1234 in the stream. Used for error reporting. 

1235 

1236 See also 

1237 -------- 

1238 lsst.pex.config.Config.load 

1239 lsst.pex.config.Config.loadFromStream 

1240 lsst.pex.config.Config.save 

1241 lsst.pex.config.Config.saveToStream 

1242 lsst.pex.config.Config.saveToString 

1243 """ 

1244 if filename is None: 1244 ↛ 1248line 1244 didn't jump to line 1248, because the condition on line 1244 was never false

1245 # try to determine the file name; a compiled string 

1246 # has attribute "co_filename", 

1247 filename = getattr(code, "co_filename", "?") 

1248 with RecordingImporter() as importer: 

1249 globals = {"__file__": filename} 

1250 local = {root: self} 

1251 exec(code, globals, local) 

1252 

1253 self._imports.update(importer.getModules()) 

1254 

1255 def save(self, filename, root="config"): 

1256 """Save a Python script to the named file, which, when loaded, 

1257 reproduces this config. 

1258 

1259 Parameters 

1260 ---------- 

1261 filename : `str` 

1262 Desination filename of this configuration. 

1263 root : `str`, optional 

1264 Name to use for the root config variable. The same value must be 

1265 used when loading (see `lsst.pex.config.Config.load`). 

1266 

1267 See also 

1268 -------- 

1269 lsst.pex.config.Config.saveToStream 

1270 lsst.pex.config.Config.saveToString 

1271 lsst.pex.config.Config.load 

1272 lsst.pex.config.Config.loadFromStream 

1273 lsst.pex.config.Config.loadFromString 

1274 """ 

1275 d = os.path.dirname(filename) 

1276 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1277 self.saveToStream(outfile, root) 

1278 # tempfile is hardcoded to create files with mode '0600' 

1279 # for an explantion of these antics see: 

1280 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1281 umask = os.umask(0o077) 

1282 os.umask(umask) 

1283 os.chmod(outfile.name, (~umask & 0o666)) 

1284 # chmod before the move so we get quasi-atomic behavior if the 

1285 # source and dest. are on the same filesystem. 

1286 # os.rename may not work across filesystems 

1287 shutil.move(outfile.name, filename) 

1288 

1289 def saveToString(self, skipImports=False): 

1290 """Return the Python script form of this configuration as an executable 

1291 string. 

1292 

1293 Parameters 

1294 ---------- 

1295 skipImports : `bool`, optional 

1296 If `True` then do not include ``import`` statements in output, 

1297 this is to support human-oriented output from ``pipetask`` where 

1298 additional clutter is not useful. 

1299 

1300 Returns 

1301 ------- 

1302 code : `str` 

1303 A code string readable by `loadFromString`. 

1304 

1305 See also 

1306 -------- 

1307 lsst.pex.config.Config.save 

1308 lsst.pex.config.Config.saveToStream 

1309 lsst.pex.config.Config.load 

1310 lsst.pex.config.Config.loadFromStream 

1311 lsst.pex.config.Config.loadFromString 

1312 """ 

1313 buffer = io.StringIO() 

1314 self.saveToStream(buffer, skipImports=skipImports) 

1315 return buffer.getvalue() 

1316 

1317 def saveToStream(self, outfile, root="config", skipImports=False): 

1318 """Save a configuration file to a stream, which, when loaded, 

1319 reproduces this config. 

1320 

1321 Parameters 

1322 ---------- 

1323 outfile : file-like object 

1324 Destination file object write the config into. Accepts strings not 

1325 bytes. 

1326 root 

1327 Name to use for the root config variable. The same value must be 

1328 used when loading (see `lsst.pex.config.Config.load`). 

1329 skipImports : `bool`, optional 

1330 If `True` then do not include ``import`` statements in output, 

1331 this is to support human-oriented output from ``pipetask`` where 

1332 additional clutter is not useful. 

1333 

1334 See also 

1335 -------- 

1336 lsst.pex.config.Config.save 

1337 lsst.pex.config.Config.saveToString 

1338 lsst.pex.config.Config.load 

1339 lsst.pex.config.Config.loadFromStream 

1340 lsst.pex.config.Config.loadFromString 

1341 """ 

1342 tmp = self._name 

1343 self._rename(root) 

1344 try: 

1345 if not skipImports: 1345 ↛ 1359line 1345 didn't jump to line 1359, because the condition on line 1345 was never false

1346 self._collectImports() 

1347 # Remove self from the set, as it is handled explicitly below 

1348 self._imports.remove(self.__module__) 

1349 configType = type(self) 

1350 typeString = _typeStr(configType) 

1351 outfile.write(f"import {configType.__module__}\n") 

1352 outfile.write( 

1353 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1354 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1355 ) 

1356 for imp in sorted(self._imports): 1356 ↛ 1357line 1356 didn't jump to line 1357, because the loop on line 1356 never started

1357 if imp in sys.modules and sys.modules[imp] is not None: 

1358 outfile.write("import {}\n".format(imp)) 

1359 self._save(outfile) 

1360 finally: 

1361 self._rename(tmp) 

1362 

1363 def freeze(self): 

1364 """Make this config, and all subconfigs, read-only.""" 

1365 self._frozen = True 

1366 for field in self._fields.values(): 

1367 field.freeze(self) 

1368 

1369 def _save(self, outfile): 

1370 """Save this config to an open stream object. 

1371 

1372 Parameters 

1373 ---------- 

1374 outfile : file-like object 

1375 Destination file object write the config into. Accepts strings not 

1376 bytes. 

1377 """ 

1378 for field in self._fields.values(): 

1379 field.save(outfile, self) 

1380 

1381 def _collectImports(self): 

1382 """Adds module containing self to the list of things to import and 

1383 then loops over all the fields in the config calling a corresponding 

1384 collect method. The field method will call _collectImports on any 

1385 configs it may own and return the set of things to import. This 

1386 returned set will be merged with the set of imports for this config 

1387 class. 

1388 """ 

1389 self._imports.add(self.__module__) 

1390 for name, field in self._fields.items(): 

1391 field._collectImports(self, self._imports) 

1392 

1393 def toDict(self): 

1394 """Make a dictionary of field names and their values. 

1395 

1396 Returns 

1397 ------- 

1398 dict_ : `dict` 

1399 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1400 Values are `~lsst.pex.config.Field` values. 

1401 

1402 See also 

1403 -------- 

1404 lsst.pex.config.Field.toDict 

1405 

1406 Notes 

1407 ----- 

1408 This method uses the `~lsst.pex.config.Field.toDict` method of 

1409 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1410 implement a ``toDict`` method for *this* method to work. 

1411 """ 

1412 dict_ = {} 

1413 for name, field in self._fields.items(): 

1414 dict_[name] = field.toDict(self) 

1415 return dict_ 

1416 

1417 def names(self): 

1418 """Get all the field names in the config, recursively. 

1419 

1420 Returns 

1421 ------- 

1422 names : `list` of `str` 

1423 Field names. 

1424 """ 

1425 # 

1426 # Rather than sort out the recursion all over again use the 

1427 # pre-existing saveToStream() 

1428 # 

1429 with io.StringIO() as strFd: 

1430 self.saveToStream(strFd, "config") 

1431 contents = strFd.getvalue() 

1432 strFd.close() 

1433 # 

1434 # Pull the names out of the dumped config 

1435 # 

1436 keys = [] 

1437 for line in contents.split("\n"): 

1438 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1439 continue 

1440 

1441 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1442 if mat: 

1443 keys.append(mat.group(1)) 

1444 

1445 return keys 

1446 

1447 def _rename(self, name): 

1448 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1449 

1450 Parameters 

1451 ---------- 

1452 name : `str` 

1453 New name for this config in its parent `~lsst.pex.config.Config`. 

1454 

1455 Notes 

1456 ----- 

1457 This method uses the `~lsst.pex.config.Field.rename` method of 

1458 individual `lsst.pex.config.Field` instances. 

1459 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1460 method for *this* method to work. 

1461 

1462 See also 

1463 -------- 

1464 lsst.pex.config.Field.rename 

1465 """ 

1466 self._name = name 

1467 for field in self._fields.values(): 

1468 field.rename(self) 

1469 

1470 def validate(self): 

1471 """Validate the Config, raising an exception if invalid. 

1472 

1473 Raises 

1474 ------ 

1475 lsst.pex.config.FieldValidationError 

1476 Raised if verification fails. 

1477 

1478 Notes 

1479 ----- 

1480 The base class implementation performs type checks on all fields by 

1481 calling their `~lsst.pex.config.Field.validate` methods. 

1482 

1483 Complex single-field validation can be defined by deriving new Field 

1484 types. For convenience, some derived `lsst.pex.config.Field`-types 

1485 (`~lsst.pex.config.ConfigField` and 

1486 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config` 

1487 that handle recursing into subconfigs. 

1488 

1489 Inter-field relationships should only be checked in derived 

1490 `~lsst.pex.config.Config` classes after calling this method, and base 

1491 validation is complete. 

1492 """ 

1493 for field in self._fields.values(): 

1494 field.validate(self) 

1495 

1496 def formatHistory(self, name, **kwargs): 

1497 """Format a configuration field's history to a human-readable format. 

1498 

1499 Parameters 

1500 ---------- 

1501 name : `str` 

1502 Name of a `~lsst.pex.config.Field` in this config. 

1503 kwargs 

1504 Keyword arguments passed to `lsst.pex.config.history.format`. 

1505 

1506 Returns 

1507 ------- 

1508 history : `str` 

1509 A string containing the formatted history. 

1510 

1511 See also 

1512 -------- 

1513 lsst.pex.config.history.format 

1514 """ 

1515 import lsst.pex.config.history as pexHist 

1516 

1517 return pexHist.format(self, name, **kwargs) 

1518 

1519 history = property(lambda x: x._history) 1519 ↛ exitline 1519 didn't run the lambda on line 1519

1520 """Read-only history. 

1521 """ 

1522 

1523 def __setattr__(self, attr, value, at=None, label="assignment"): 

1524 """Set an attribute (such as a field's value). 

1525 

1526 Notes 

1527 ----- 

1528 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1529 locked such that no additional attributes nor properties may be added 

1530 to them dynamically. 

1531 

1532 Although this is not the standard Python behavior, it helps to protect 

1533 users from accidentally mispelling a field name, or trying to set a 

1534 non-existent field. 

1535 """ 

1536 if attr in self._fields: 

1537 if self._fields[attr].deprecated is not None: 1537 ↛ 1538line 1537 didn't jump to line 1538, because the condition on line 1537 was never true

1538 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1539 warnings.warn( 

1540 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1541 FutureWarning, 

1542 stacklevel=2, 

1543 ) 

1544 if at is None: 1544 ↛ 1547line 1544 didn't jump to line 1547, because the condition on line 1544 was never false

1545 at = getCallStack() 

1546 # This allows Field descriptors to work. 

1547 self._fields[attr].__set__(self, value, at=at, label=label) 

1548 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1548 ↛ 1550line 1548 didn't jump to line 1550, because the condition on line 1548 was never true

1549 # This allows properties and other non-Field descriptors to work. 

1550 return object.__setattr__(self, attr, value) 

1551 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1551 ↛ 1556line 1551 didn't jump to line 1556, because the condition on line 1551 was never false

1552 # This allows specific private attributes to work. 

1553 self.__dict__[attr] = value 

1554 else: 

1555 # We throw everything else. 

1556 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr)) 

1557 

1558 def __delattr__(self, attr, at=None, label="deletion"): 

1559 if attr in self._fields: 

1560 if at is None: 

1561 at = getCallStack() 

1562 self._fields[attr].__delete__(self, at=at, label=label) 

1563 else: 

1564 object.__delattr__(self, attr) 

1565 

1566 def __eq__(self, other): 

1567 if type(other) == type(self): 1567 ↛ 1568line 1567 didn't jump to line 1568, because the condition on line 1567 was never true

1568 for name in self._fields: 

1569 thisValue = getattr(self, name) 

1570 otherValue = getattr(other, name) 

1571 if isinstance(thisValue, float) and math.isnan(thisValue): 

1572 if not math.isnan(otherValue): 

1573 return False 

1574 elif thisValue != otherValue: 

1575 return False 

1576 return True 

1577 return False 

1578 

1579 def __ne__(self, other): 

1580 return not self.__eq__(other) 

1581 

1582 def __str__(self): 

1583 return str(self.toDict()) 

1584 

1585 def __repr__(self): 

1586 return "%s(%s)" % ( 

1587 _typeStr(self), 

1588 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None), 

1589 ) 

1590 

1591 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1592 """Compare this configuration to another `~lsst.pex.config.Config` for 

1593 equality. 

1594 

1595 Parameters 

1596 ---------- 

1597 other : `lsst.pex.config.Config` 

1598 Other `~lsst.pex.config.Config` object to compare against this 

1599 config. 

1600 shortcut : `bool`, optional 

1601 If `True`, return as soon as an inequality is found. Default is 

1602 `True`. 

1603 rtol : `float`, optional 

1604 Relative tolerance for floating point comparisons. 

1605 atol : `float`, optional 

1606 Absolute tolerance for floating point comparisons. 

1607 output : callable, optional 

1608 A callable that takes a string, used (possibly repeatedly) to 

1609 report inequalities. 

1610 

1611 Returns 

1612 ------- 

1613 isEqual : `bool` 

1614 `True` when the two `lsst.pex.config.Config` instances are equal. 

1615 `False` if there is an inequality. 

1616 

1617 See also 

1618 -------- 

1619 lsst.pex.config.compareConfigs 

1620 

1621 Notes 

1622 ----- 

1623 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1624 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1625 are not considered by this method. 

1626 

1627 Floating point comparisons are performed by `numpy.allclose`. 

1628 """ 

1629 name1 = self._name if self._name is not None else "config" 

1630 name2 = other._name if other._name is not None else "config" 

1631 name = getComparisonName(name1, name2) 

1632 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1633 

1634 @classmethod 

1635 def __init_subclass__(cls, **kwargs): 

1636 """Run initialization for every subclass. 

1637 

1638 Specifically registers the subclass with a YAML representer 

1639 and YAML constructor (if pyyaml is available) 

1640 """ 

1641 super().__init_subclass__(**kwargs) 

1642 

1643 if not yaml: 1643 ↛ 1644line 1643 didn't jump to line 1644, because the condition on line 1643 was never true

1644 return 

1645 

1646 yaml.add_representer(cls, _yaml_config_representer) 

1647 

1648 @classmethod 

1649 def _fromPython(cls, config_py): 

1650 """Instantiate a `Config`-subclass from serialized Python form. 

1651 

1652 Parameters 

1653 ---------- 

1654 config_py : `str` 

1655 A serialized form of the Config as created by 

1656 `Config.saveToStream`. 

1657 

1658 Returns 

1659 ------- 

1660 config : `Config` 

1661 Reconstructed `Config` instant. 

1662 """ 

1663 cls = _classFromPython(config_py) 

1664 return unreduceConfig(cls, config_py) 

1665 

1666 

1667def _classFromPython(config_py): 

1668 """Return the Config subclass required by this Config serialization. 

1669 

1670 Parameters 

1671 ---------- 

1672 config_py : `str` 

1673 A serialized form of the Config as created by 

1674 `Config.saveToStream`. 

1675 

1676 Returns 

1677 ------- 

1678 cls : `type` 

1679 The `Config` subclass associated with this config. 

1680 """ 

1681 # standard serialization has the form: 

1682 # import config.class 

1683 # assert type(config)==config.class.Config, ... 

1684 # We want to parse these two lines so we can get the class itself 

1685 

1686 # Do a single regex to avoid large string copies when splitting a 

1687 # large config into separate lines. 

1688 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1689 

1690 if not matches: 

1691 first_line, second_line, _ = config_py.split("\n", 2) 

1692 raise ValueError( 

1693 "First two lines did not match expected form. Got:\n" f" - {first_line}\n" f" - {second_line}" 

1694 ) 

1695 

1696 module_name = matches.group(1) 

1697 module = importlib.import_module(module_name) 

1698 

1699 # Second line 

1700 full_name = matches.group(2) 

1701 

1702 # Remove the module name from the full name 

1703 if not full_name.startswith(module_name): 

1704 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1705 

1706 # if module name is a.b.c and full name is a.b.c.d.E then 

1707 # we need to remove a.b.c. and iterate over the remainder 

1708 # The +1 is for the extra dot after a.b.c 

1709 remainder = full_name[len(module_name) + 1 :] 

1710 components = remainder.split(".") 

1711 pytype = module 

1712 for component in components: 

1713 pytype = getattr(pytype, component) 

1714 return pytype 

1715 

1716 

1717def unreduceConfig(cls, stream): 

1718 """Create a `~lsst.pex.config.Config` from a stream. 

1719 

1720 Parameters 

1721 ---------- 

1722 cls : `lsst.pex.config.Config`-type 

1723 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1724 with configurations in the ``stream``. 

1725 stream : file-like object, `str`, or compiled string 

1726 Stream containing configuration override code. 

1727 

1728 Returns 

1729 ------- 

1730 config : `lsst.pex.config.Config` 

1731 Config instance. 

1732 

1733 See also 

1734 -------- 

1735 lsst.pex.config.Config.loadFromStream 

1736 """ 

1737 config = cls() 

1738 config.loadFromStream(stream) 

1739 return config