Coverage for python/lsst/pex/config/config.py: 58%

458 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-06 02:38 -0700

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from typing import Any, ForwardRef, Generic, Mapping, Optional, TypeVar, Union, cast, overload 

49 

50try: 

51 from types import GenericAlias 

52except ImportError: 

53 # cover python 3.8 usage 

54 GenericAlias = type(Mapping[int, int]) 

55 

56# if YAML is not available that's fine and we simply don't register 

57# the yaml representer since we know it won't be used. 

58try: 

59 import yaml 

60except ImportError: 

61 yaml = None 

62 

63from .callStack import getCallStack, getStackFrame 

64from .comparison import compareConfigs, compareScalars, getComparisonName 

65 

66if yaml: 66 ↛ 77line 66 didn't jump to line 77, because the condition on line 66 was never false

67 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

68 

69 try: 

70 # CLoader is not always available 

71 from yaml import CLoader 

72 

73 YamlLoaders += (CLoader,) 

74 except ImportError: 

75 pass 

76else: 

77 YamlLoaders = () 

78 doImport = None 

79 

80 

81if int(sys.version_info.minor) < 9: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true

82 genericAliasKwds = {"_root": True} 

83else: 

84 genericAliasKwds = {} 

85 

86 

87class _PexConfigGenericAlias(GenericAlias, **genericAliasKwds): 

88 """A Subclass of python's GenericAlias used in defining and instantiating 

89 Generics. 

90 

91 This class differs from `types.GenericAlias` in that it calls a method 

92 named _parseTypingArgs defined on Fields. This method gives Field and its 

93 subclasses an opportunity to transform type parameters into class key word 

94 arguments. Code authors do not need to implement any returns of this object 

95 directly, and instead only need implement _parseTypingArgs, if a Field 

96 subclass differs from the base class implementation. 

97 

98 This class is intended to be an implementation detail, returned from a 

99 Field's `__class_getitem__` method. 

100 """ 

101 

102 def __call__(self, *args: Any, **kwds: Any) -> Any: 

103 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

104 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

105 

106 

107FieldTypeVar = TypeVar("FieldTypeVar") 

108 

109 

110class UnexpectedProxyUsageError(TypeError): 

111 """Exception raised when a proxy class is used in a context that suggests 

112 it should have already been converted to the thing it proxies. 

113 """ 

114 

115 

116def _joinNamePath(prefix=None, name=None, index=None): 

117 """Generate nested configuration names.""" 

118 if not prefix and not name: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 raise ValueError("Invalid name: cannot be None") 

120 elif not name: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 name = prefix 

122 elif prefix and name: 122 ↛ 125line 122 didn't jump to line 125, because the condition on line 122 was never false

123 name = prefix + "." + name 

124 

125 if index is not None: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true

126 return "%s[%r]" % (name, index) 

127 else: 

128 return name 

129 

130 

131def _autocast(x, dtype): 

132 """Cast a value to a type, if appropriate. 

133 

134 Parameters 

135 ---------- 

136 x : object 

137 A value. 

138 dtype : tpye 

139 Data type, such as `float`, `int`, or `str`. 

140 

141 Returns 

142 ------- 

143 values : object 

144 If appropriate, the returned value is ``x`` cast to the given type 

145 ``dtype``. If the cast cannot be performed the original value of 

146 ``x`` is returned. 

147 """ 

148 if dtype == float and isinstance(x, int): 

149 return float(x) 

150 return x 

151 

152 

153def _typeStr(x): 

154 """Generate a fully-qualified type name. 

155 

156 Returns 

157 ------- 

158 `str` 

159 Fully-qualified type name. 

160 

161 Notes 

162 ----- 

163 This function is used primarily for writing config files to be executed 

164 later upon with the 'load' function. 

165 """ 

166 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

167 xtype = x 

168 else: 

169 xtype = type(x) 

170 if (sys.version_info.major <= 2 and xtype.__module__ == "__builtin__") or xtype.__module__ == "builtins": 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 return xtype.__name__ 

172 else: 

173 return "%s.%s" % (xtype.__module__, xtype.__name__) 

174 

175 

176if yaml: 176 ↛ 209line 176 didn't jump to line 209, because the condition on line 176 was never false

177 

178 def _yaml_config_representer(dumper, data): 

179 """Represent a Config object in a form suitable for YAML. 

180 

181 Stores the serialized stream as a scalar block string. 

182 """ 

183 stream = io.StringIO() 

184 data.saveToStream(stream) 

185 config_py = stream.getvalue() 

186 

187 # Strip multiple newlines from the end of the config 

188 # This simplifies the YAML to use | and not |+ 

189 config_py = config_py.rstrip() + "\n" 

190 

191 # Trailing spaces force pyyaml to use non-block form. 

192 # Remove the trailing spaces so it has no choice 

193 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

194 

195 # Store the Python as a simple scalar 

196 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

197 

198 def _yaml_config_constructor(loader, node): 

199 """Construct a config from YAML""" 

200 config_py = loader.construct_scalar(node) 

201 return Config._fromPython(config_py) 

202 

203 # Register a generic constructor for Config and all subclasses 

204 # Need to register for all the loaders we would like to use 

205 for loader in YamlLoaders: 

206 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

207 

208 

209class ConfigMeta(type): 

210 """A metaclass for `lsst.pex.config.Config`. 

211 

212 Notes 

213 ----- 

214 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

215 class attributes as a class attribute called ``_fields``, and adds 

216 the name of each field as an instance variable of the field itself (so you 

217 don't have to pass the name of the field to the field constructor). 

218 """ 

219 

220 def __init__(cls, name, bases, dict_): 

221 type.__init__(cls, name, bases, dict_) 

222 cls._fields = {} 

223 cls._source = getStackFrame() 

224 

225 def getFields(classtype): 

226 fields = {} 

227 bases = list(classtype.__bases__) 

228 bases.reverse() 

229 for b in bases: 

230 fields.update(getFields(b)) 

231 

232 for k, v in classtype.__dict__.items(): 

233 if isinstance(v, Field): 

234 fields[k] = v 

235 return fields 

236 

237 fields = getFields(cls) 

238 for k, v in fields.items(): 

239 setattr(cls, k, copy.deepcopy(v)) 

240 

241 def __setattr__(cls, name, value): 

242 if isinstance(value, Field): 

243 value.name = name 

244 cls._fields[name] = value 

245 type.__setattr__(cls, name, value) 

246 

247 

248class FieldValidationError(ValueError): 

249 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

250 particular ``~lsst.pex.config.Config``. 

251 

252 Parameters 

253 ---------- 

254 field : `lsst.pex.config.Field` 

255 The field that was not valid. 

256 config : `lsst.pex.config.Config` 

257 The config containing the invalid field. 

258 msg : `str` 

259 Text describing why the field was not valid. 

260 """ 

261 

262 def __init__(self, field, config, msg): 

263 self.fieldType = type(field) 

264 """Type of the `~lsst.pex.config.Field` that incurred the error. 

265 """ 

266 

267 self.fieldName = field.name 

268 """Name of the `~lsst.pex.config.Field` instance that incurred the 

269 error (`str`). 

270 

271 See also 

272 -------- 

273 lsst.pex.config.Field.name 

274 """ 

275 

276 self.fullname = _joinNamePath(config._name, field.name) 

277 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

278 (`str`). 

279 """ 

280 

281 self.history = config.history.setdefault(field.name, []) 

282 """Full history of all changes to the `~lsst.pex.config.Field` 

283 instance. 

284 """ 

285 

286 self.fieldSource = field.source 

287 """File and line number of the `~lsst.pex.config.Field` definition. 

288 """ 

289 

290 self.configSource = config._source 

291 error = ( 

292 "%s '%s' failed validation: %s\n" 

293 "For more information see the Field definition at:\n%s" 

294 " and the Config definition at:\n%s" 

295 % ( 

296 self.fieldType.__name__, 

297 self.fullname, 

298 msg, 

299 self.fieldSource.format(), 

300 self.configSource.format(), 

301 ) 

302 ) 

303 super().__init__(error) 

304 

305 

306class Field(Generic[FieldTypeVar]): 

307 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

308 `complex`, `bool`, and `str` data types. 

309 

310 Parameters 

311 ---------- 

312 doc : `str` 

313 A description of the field for users. 

314 dtype : type, optional 

315 The field's data type. ``Field`` only supports basic data types: 

316 `int`, `float`, `complex`, `bool`, and `str`. See 

317 `Field.supportedTypes`. Optional if supplied as a typing argument to 

318 the class. 

319 default : object, optional 

320 The field's default value. 

321 check : callable, optional 

322 A callable that is called with the field's value. This callable should 

323 return `False` if the value is invalid. More complex inter-field 

324 validation can be written as part of the 

325 `lsst.pex.config.Config.validate` method. 

326 optional : `bool`, optional 

327 This sets whether the field is considered optional, and therefore 

328 doesn't need to be set by the user. When `False`, 

329 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

330 deprecated : None or `str`, optional 

331 A description of why this Field is deprecated, including removal date. 

332 If not None, the string is appended to the docstring for this Field. 

333 

334 Raises 

335 ------ 

336 ValueError 

337 Raised when the ``dtype`` parameter is not one of the supported types 

338 (see `Field.supportedTypes`). 

339 

340 See also 

341 -------- 

342 ChoiceField 

343 ConfigChoiceField 

344 ConfigDictField 

345 ConfigField 

346 ConfigurableField 

347 DictField 

348 ListField 

349 RangeField 

350 RegistryField 

351 

352 Notes 

353 ----- 

354 ``Field`` instances (including those of any subclass of ``Field``) are used 

355 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

356 example, below). ``Field`` attributes work like the `property` attributes 

357 of classes that implement custom setters and getters. `Field` attributes 

358 belong to the class, but operate on the instance. Formally speaking, 

359 `Field` attributes are `descriptors 

360 <https://docs.python.org/3/howto/descriptor.html>`_. 

361 

362 When you access a `Field` attribute on a `Config` instance, you don't 

363 get the `Field` instance itself. Instead, you get the value of that field, 

364 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

365 container type (like a `lsst.pex.config.List`) depending on the field's 

366 type. See the example, below. 

367 

368 Fields can be annotated with a type similar to other python classes (python 

369 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

370 See the name field in the Config example below for an example of this. 

371 Unlike most other uses in python, this has an effect at type checking *and* 

372 runtime. If the type is specified with a class annotation, it will be used 

373 as the value of the ``dtype`` in the ``Field`` and there is no need to 

374 specify it as an argument during instantiation. 

375 

376 There are Some notes on dtype through type annotation syntax. Type 

377 annotation syntax supports supplying the argument as a string of a type 

378 name. i.e. "float", but this cannot be used to resolve circular references. 

379 Type annotation syntax can be used on an identifier in addition to Class 

380 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

381 ``someField = Field[str](doc="some doc"). However, this syntax is only 

382 useful for annotating the type of the identifier (i.e. variable in previous 

383 example) and does nothing for assigning the dtype of the ``Field``. 

384 

385 

386 Examples 

387 -------- 

388 Instances of ``Field`` should be used as class attributes of 

389 `lsst.pex.config.Config` subclasses: 

390 

391 >>> from lsst.pex.config import Config, Field 

392 >>> class Example(Config): 

393 ... myInt = Field("An integer field.", int, default=0) 

394 ... name = Field[str](doc="A string Field") 

395 ... 

396 >>> print(config.myInt) 

397 0 

398 >>> config.myInt = 5 

399 >>> print(config.myInt) 

400 5 

401 """ 

402 

403 name: str 

404 """Identifier (variable name) used to refer to a Field within a Config 

405 Class. 

406 """ 

407 

408 supportedTypes = set((str, bool, float, int, complex)) 

409 """Supported data types for field values (`set` of types). 

410 """ 

411 

412 @staticmethod 

413 def _parseTypingArgs( 

414 params: Union[tuple[type, ...], tuple[str, ...]], kwds: Mapping[str, Any] 

415 ) -> Mapping[str, Any]: 

416 """Parses type annotations into keyword constructor arguments. 

417 

418 This is a special private method that interprets type arguments (i.e. 

419 Field[str]) into keyword arguments to be passed on to the constructor. 

420 

421 Subclasses of Field can implement this method to customize how they 

422 handle turning type parameters into keyword arguments (see DictField 

423 for an example) 

424 

425 Parameters 

426 ---------- 

427 params : `tuple` of `type` or `tuple` of str 

428 Parameters passed to the type annotation. These will either be 

429 types or strings. Strings are to interpreted as forward references 

430 and will be treated as such. 

431 kwds : `MutableMapping` with keys of `str` and values of `Any` 

432 These are the user supplied keywords that are to be passed to the 

433 Field constructor. 

434 

435 Returns 

436 ------- 

437 kwds : `MutableMapping` with keys of `str` and values of `Any` 

438 The mapping of keywords that will be passed onto the constructor 

439 of the Field. Should be filled in with any information gleaned 

440 from the input parameters. 

441 

442 Raises 

443 ------ 

444 ValueError : 

445 Raised if params is of incorrect length. 

446 Raised if a forward reference could not be resolved 

447 Raised if there is a conflict between params and values in kwds 

448 """ 

449 if len(params) > 1: 

450 raise ValueError("Only single type parameters are supported") 

451 unpackedParams = params[0] 

452 if isinstance(unpackedParams, str): 

453 _typ = ForwardRef(unpackedParams) 

454 # type ignore below because typeshed seems to be wrong. It 

455 # indicates there are only 2 args, as it was in python 3.8, but 

456 # 3.9+ takes 3 args. Attempt in old style and new style to 

457 # work with both. 

458 try: 

459 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

460 except TypeError: 

461 # python 3.8 path 

462 result = _typ._evaluate(globals(), locals()) 

463 if result is None: 

464 raise ValueError("Could not deduce type from input") 

465 unpackedParams = cast(type, result) 

466 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

467 raise ValueError("Conflicting definition for dtype") 

468 elif "dtype" not in kwds: 

469 kwds = {**kwds, **{"dtype": unpackedParams}} 

470 return kwds 

471 

472 def __class_getitem__(cls, params: Union[tuple[type, ...], type, ForwardRef]): 

473 return _PexConfigGenericAlias(cls, params) 

474 

475 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

476 if dtype is None: 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 raise ValueError( 

478 "dtype must either be supplied as an argument or as a type argument to the class" 

479 ) 

480 if dtype not in self.supportedTypes: 480 ↛ 481line 480 didn't jump to line 481, because the condition on line 480 was never true

481 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

482 

483 source = getStackFrame() 

484 self._setup( 

485 doc=doc, 

486 dtype=dtype, 

487 default=default, 

488 check=check, 

489 optional=optional, 

490 source=source, 

491 deprecated=deprecated, 

492 ) 

493 

494 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

495 """Set attributes, usually during initialization.""" 

496 self.dtype = dtype 

497 """Data type for the field. 

498 """ 

499 

500 # append the deprecation message to the docstring. 

501 if deprecated is not None: 

502 doc = f"{doc} Deprecated: {deprecated}" 

503 self.doc = doc 

504 """A description of the field (`str`). 

505 """ 

506 

507 self.deprecated = deprecated 

508 """If not None, a description of why this field is deprecated (`str`). 

509 """ 

510 

511 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

512 if optional or default is not None: 

513 self.__doc__ += f", default ``{default!r}``" 

514 self.__doc__ += ")" 

515 

516 self.default = default 

517 """Default value for this field. 

518 """ 

519 

520 self.check = check 

521 """A user-defined function that validates the value of the field. 

522 """ 

523 

524 self.optional = optional 

525 """Flag that determines if the field is required to be set (`bool`). 

526 

527 When `False`, `lsst.pex.config.Config.validate` will fail if the 

528 field's value is `None`. 

529 """ 

530 

531 self.source = source 

532 """The stack frame where this field is defined (`list` of 

533 `lsst.pex.config.callStack.StackFrame`). 

534 """ 

535 

536 def rename(self, instance): 

537 """Rename the field in a `~lsst.pex.config.Config` (for internal use 

538 only). 

539 

540 Parameters 

541 ---------- 

542 instance : `lsst.pex.config.Config` 

543 The config instance that contains this field. 

544 

545 Notes 

546 ----- 

547 This method is invoked by the `lsst.pex.config.Config` object that 

548 contains this field and should not be called directly. 

549 

550 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

551 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should 

552 rename each subconfig with the full field name as generated by 

553 `lsst.pex.config.config._joinNamePath`. 

554 """ 

555 pass 

556 

557 def validate(self, instance): 

558 """Validate the field (for internal use only). 

559 

560 Parameters 

561 ---------- 

562 instance : `lsst.pex.config.Config` 

563 The config instance that contains this field. 

564 

565 Raises 

566 ------ 

567 lsst.pex.config.FieldValidationError 

568 Raised if verification fails. 

569 

570 Notes 

571 ----- 

572 This method provides basic validation: 

573 

574 - Ensures that the value is not `None` if the field is not optional. 

575 - Ensures type correctness. 

576 - Ensures that the user-provided ``check`` function is valid. 

577 

578 Most `~lsst.pex.config.Field` subclasses should call 

579 `lsst.pex.config.field.Field.validate` if they re-implement 

580 `~lsst.pex.config.field.Field.validate`. 

581 """ 

582 value = self.__get__(instance) 

583 if not self.optional and value is None: 

584 raise FieldValidationError(self, instance, "Required value cannot be None") 

585 

586 def freeze(self, instance): 

587 """Make this field read-only (for internal use only). 

588 

589 Parameters 

590 ---------- 

591 instance : `lsst.pex.config.Config` 

592 The config instance that contains this field. 

593 

594 Notes 

595 ----- 

596 Freezing is only relevant for fields that hold subconfigs. Fields which 

597 hold subconfigs should freeze each subconfig. 

598 

599 **Subclasses should implement this method.** 

600 """ 

601 pass 

602 

603 def _validateValue(self, value): 

604 """Validate a value. 

605 

606 Parameters 

607 ---------- 

608 value : object 

609 The value being validated. 

610 

611 Raises 

612 ------ 

613 TypeError 

614 Raised if the value's type is incompatible with the field's 

615 ``dtype``. 

616 ValueError 

617 Raised if the value is rejected by the ``check`` method. 

618 """ 

619 if value is None: 619 ↛ 620line 619 didn't jump to line 620, because the condition on line 619 was never true

620 return 

621 

622 if not isinstance(value, self.dtype): 622 ↛ 623line 622 didn't jump to line 623, because the condition on line 622 was never true

623 msg = "Value %s is of incorrect type %s. Expected type %s" % ( 

624 value, 

625 _typeStr(value), 

626 _typeStr(self.dtype), 

627 ) 

628 raise TypeError(msg) 

629 if self.check is not None and not self.check(value): 629 ↛ 630line 629 didn't jump to line 630, because the condition on line 629 was never true

630 msg = "Value %s is not a valid value" % str(value) 

631 raise ValueError(msg) 

632 

633 def _collectImports(self, instance, imports): 

634 """This function should call the _collectImports method on all config 

635 objects the field may own, and union them with the supplied imports 

636 set. 

637 

638 Parameters 

639 ---------- 

640 instance : instance or subclass of `lsst.pex.config.Config` 

641 A config object that has this field defined on it 

642 imports : `set` 

643 Set of python modules that need imported after persistence 

644 """ 

645 pass 

646 

647 def save(self, outfile, instance): 

648 """Save this field to a file (for internal use only). 

649 

650 Parameters 

651 ---------- 

652 outfile : file-like object 

653 A writeable field handle. 

654 instance : `Config` 

655 The `Config` instance that contains this field. 

656 

657 Notes 

658 ----- 

659 This method is invoked by the `~lsst.pex.config.Config` object that 

660 contains this field and should not be called directly. 

661 

662 The output consists of the documentation string 

663 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

664 line is formatted as an assignment: ``{fullname}={value}``. 

665 

666 This output can be executed with Python. 

667 """ 

668 value = self.__get__(instance) 

669 fullname = _joinNamePath(instance._name, self.name) 

670 

671 if self.deprecated and value == self.default: 671 ↛ 672line 671 didn't jump to line 672, because the condition on line 671 was never true

672 return 

673 

674 # write full documentation string as comment lines 

675 # (i.e. first character is #) 

676 doc = "# " + str(self.doc).replace("\n", "\n# ") 

677 if isinstance(value, float) and not math.isfinite(value): 677 ↛ 679line 677 didn't jump to line 679, because the condition on line 677 was never true

678 # non-finite numbers need special care 

679 outfile.write("{}\n{}=float('{!r}')\n\n".format(doc, fullname, value)) 

680 else: 

681 outfile.write("{}\n{}={!r}\n\n".format(doc, fullname, value)) 

682 

683 def toDict(self, instance): 

684 """Convert the field value so that it can be set as the value of an 

685 item in a `dict` (for internal use only). 

686 

687 Parameters 

688 ---------- 

689 instance : `Config` 

690 The `Config` that contains this field. 

691 

692 Returns 

693 ------- 

694 value : object 

695 The field's value. See *Notes*. 

696 

697 Notes 

698 ----- 

699 This method invoked by the owning `~lsst.pex.config.Config` object and 

700 should not be called directly. 

701 

702 Simple values are passed through. Complex data structures must be 

703 manipulated. For example, a `~lsst.pex.config.Field` holding a 

704 subconfig should, instead of the subconfig object, return a `dict` 

705 where the keys are the field names in the subconfig, and the values are 

706 the field values in the subconfig. 

707 """ 

708 return self.__get__(instance) 

709 

710 @overload 

711 def __get__( 

712 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

713 ) -> "Field[FieldTypeVar]": 

714 ... 

715 

716 @overload 

717 def __get__( 

718 self, instance: "Config", owner: Any = None, at: Any = None, label: str = "default" 

719 ) -> FieldTypeVar: 

720 ... 

721 

722 def __get__(self, instance, owner=None, at=None, label="default"): 

723 """Define how attribute access should occur on the Config instance 

724 This is invoked by the owning config object and should not be called 

725 directly 

726 

727 When the field attribute is accessed on a Config class object, it 

728 returns the field object itself in order to allow inspection of 

729 Config classes. 

730 

731 When the field attribute is access on a config instance, the actual 

732 value described by the field (and held by the Config instance) is 

733 returned. 

734 """ 

735 if instance is None: 735 ↛ 736line 735 didn't jump to line 736, because the condition on line 735 was never true

736 return self 

737 else: 

738 # try statements are almost free in python if they succeed 

739 try: 

740 return instance._storage[self.name] 

741 except AttributeError: 

742 if not isinstance(instance, Config): 

743 return self 

744 else: 

745 raise AttributeError( 

746 f"Config {instance} is missing _storage attribute, likely incorrectly initialized" 

747 ) 

748 

749 def __set__( 

750 self, instance: "Config", value: Optional[FieldTypeVar], at: Any = None, label: str = "assignment" 

751 ) -> None: 

752 """Set an attribute on the config instance. 

753 

754 Parameters 

755 ---------- 

756 instance : `lsst.pex.config.Config` 

757 The config instance that contains this field. 

758 value : obj 

759 Value to set on this field. 

760 at : `list` of `lsst.pex.config.callStack.StackFrame` 

761 The call stack (created by 

762 `lsst.pex.config.callStack.getCallStack`). 

763 label : `str`, optional 

764 Event label for the history. 

765 

766 Notes 

767 ----- 

768 This method is invoked by the owning `lsst.pex.config.Config` object 

769 and should not be called directly. 

770 

771 Derived `~lsst.pex.config.Field` classes may need to override the 

772 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

773 should follow the following rules: 

774 

775 - Do not allow modification of frozen configs. 

776 - Validate the new value **before** modifying the field. Except if the 

777 new value is `None`. `None` is special and no attempt should be made 

778 to validate it until `lsst.pex.config.Config.validate` is called. 

779 - Do not modify the `~lsst.pex.config.Config` instance to contain 

780 invalid values. 

781 - If the field is modified, update the history of the 

782 `lsst.pex.config.field.Field` to reflect the changes. 

783 

784 In order to decrease the need to implement this method in derived 

785 `~lsst.pex.config.Field` types, value validation is performed in the 

786 `lsst.pex.config.Field._validateValue`. If only the validation step 

787 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

788 implement `lsst.pex.config.Field._validateValue` than to reimplement 

789 ``__set__``. More complicated behavior, however, may require 

790 reimplementation. 

791 """ 

792 if instance._frozen: 792 ↛ 793line 792 didn't jump to line 793, because the condition on line 792 was never true

793 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

794 

795 history = instance._history.setdefault(self.name, []) 

796 if value is not None: 796 ↛ 803line 796 didn't jump to line 803, because the condition on line 796 was never false

797 value = _autocast(value, self.dtype) 

798 try: 

799 self._validateValue(value) 

800 except BaseException as e: 

801 raise FieldValidationError(self, instance, str(e)) 

802 

803 instance._storage[self.name] = value 

804 if at is None: 804 ↛ 805line 804 didn't jump to line 805, because the condition on line 804 was never true

805 at = getCallStack() 

806 history.append((value, at, label)) 

807 

808 def __delete__(self, instance, at=None, label="deletion"): 

809 """Delete an attribute from a `lsst.pex.config.Config` instance. 

810 

811 Parameters 

812 ---------- 

813 instance : `lsst.pex.config.Config` 

814 The config instance that contains this field. 

815 at : `list` of `lsst.pex.config.callStack.StackFrame` 

816 The call stack (created by 

817 `lsst.pex.config.callStack.getCallStack`). 

818 label : `str`, optional 

819 Event label for the history. 

820 

821 Notes 

822 ----- 

823 This is invoked by the owning `~lsst.pex.config.Config` object and 

824 should not be called directly. 

825 """ 

826 if at is None: 

827 at = getCallStack() 

828 self.__set__(instance, None, at=at, label=label) 

829 

830 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

831 """Compare a field (named `Field.name`) in two 

832 `~lsst.pex.config.Config` instances for equality. 

833 

834 Parameters 

835 ---------- 

836 instance1 : `lsst.pex.config.Config` 

837 Left-hand side `Config` instance to compare. 

838 instance2 : `lsst.pex.config.Config` 

839 Right-hand side `Config` instance to compare. 

840 shortcut : `bool`, optional 

841 **Unused.** 

842 rtol : `float`, optional 

843 Relative tolerance for floating point comparisons. 

844 atol : `float`, optional 

845 Absolute tolerance for floating point comparisons. 

846 output : callable, optional 

847 A callable that takes a string, used (possibly repeatedly) to 

848 report inequalities. 

849 

850 Notes 

851 ----- 

852 This method must be overridden by more complex `Field` subclasses. 

853 

854 See also 

855 -------- 

856 lsst.pex.config.compareScalars 

857 """ 

858 v1 = getattr(instance1, self.name) 

859 v2 = getattr(instance2, self.name) 

860 name = getComparisonName( 

861 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

862 ) 

863 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

864 

865 

866class RecordingImporter: 

867 """Importer (for `sys.meta_path`) that records which modules are being 

868 imported. 

869 

870 *This class does not do any importing itself.* 

871 

872 Examples 

873 -------- 

874 Use this class as a context manager to ensure it is properly uninstalled 

875 when done: 

876 

877 >>> with RecordingImporter() as importer: 

878 ... # import stuff 

879 ... import numpy as np 

880 ... print("Imported: " + importer.getModules()) 

881 """ 

882 

883 def __init__(self): 

884 self._modules = set() 

885 

886 def __enter__(self): 

887 self.origMetaPath = sys.meta_path 

888 sys.meta_path = [self] + sys.meta_path # type: ignore 

889 return self 

890 

891 def __exit__(self, *args): 

892 self.uninstall() 

893 return False # Don't suppress exceptions 

894 

895 def uninstall(self): 

896 """Uninstall the importer.""" 

897 sys.meta_path = self.origMetaPath 

898 

899 def find_module(self, fullname, path=None): 

900 """Called as part of the ``import`` chain of events.""" 

901 self._modules.add(fullname) 

902 # Return None because we don't do any importing. 

903 return None 

904 

905 def getModules(self): 

906 """Get the set of modules that were imported. 

907 

908 Returns 

909 ------- 

910 modules : `set` of `str` 

911 Set of imported module names. 

912 """ 

913 return self._modules 

914 

915 

916# type ignore because type checker thinks ConfigMeta is Generic when it is not 

917class Config(metaclass=ConfigMeta): # type: ignore 

918 """Base class for configuration (*config*) objects. 

919 

920 Notes 

921 ----- 

922 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

923 instances as class attributes. These are used to define most of the base 

924 class behavior. 

925 

926 ``Config`` implements a mapping API that provides many `dict`-like methods, 

927 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and 

928 `itervalues`. ``Config`` instances also support the ``in`` operator to 

929 test if a field is in the config. Unlike a `dict`, ``Config`` classes are 

930 not subscriptable. Instead, access individual fields as attributes of the 

931 configuration instance. 

932 

933 Examples 

934 -------- 

935 Config classes are subclasses of ``Config`` that have 

936 `~lsst.pex.config.Field` instances (or instances of 

937 `~lsst.pex.config.Field` subclasses) as class attributes: 

938 

939 >>> from lsst.pex.config import Config, Field, ListField 

940 >>> class DemoConfig(Config): 

941 ... intField = Field(doc="An integer field", dtype=int, default=42) 

942 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

943 ... default=['coffee', 'green tea', 'water']) 

944 ... 

945 >>> config = DemoConfig() 

946 

947 Configs support many `dict`-like APIs: 

948 

949 >>> config.keys() 

950 ['intField', 'listField'] 

951 >>> 'intField' in config 

952 True 

953 

954 Individual fields can be accessed as attributes of the configuration: 

955 

956 >>> config.intField 

957 42 

958 >>> config.listField.append('earl grey tea') 

959 >>> print(config.listField) 

960 ['coffee', 'green tea', 'water', 'earl grey tea'] 

961 """ 

962 

963 _storage: dict[str, Any] 

964 _fields: dict[str, Field] 

965 _history: dict[str, list[Any]] 

966 _imports: set[Any] 

967 

968 def __iter__(self): 

969 """Iterate over fields.""" 

970 return self._fields.__iter__() 

971 

972 def keys(self): 

973 """Get field names. 

974 

975 Returns 

976 ------- 

977 names : `dict_keys` 

978 List of `lsst.pex.config.Field` names. 

979 

980 See also 

981 -------- 

982 lsst.pex.config.Config.iterkeys 

983 """ 

984 return self._storage.keys() 

985 

986 def values(self): 

987 """Get field values. 

988 

989 Returns 

990 ------- 

991 values : `dict_values` 

992 Iterator of field values. 

993 """ 

994 return self._storage.values() 

995 

996 def items(self): 

997 """Get configurations as ``(field name, field value)`` pairs. 

998 

999 Returns 

1000 ------- 

1001 items : `dict_items` 

1002 Iterator of tuples for each configuration. Tuple items are: 

1003 

1004 0. Field name. 

1005 1. Field value. 

1006 """ 

1007 return self._storage.items() 

1008 

1009 def __contains__(self, name): 

1010 """!Return True if the specified field exists in this config 

1011 

1012 @param[in] name field name to test for 

1013 """ 

1014 return self._storage.__contains__(name) 

1015 

1016 def __new__(cls, *args, **kw): 

1017 """Allocate a new `lsst.pex.config.Config` object. 

1018 

1019 In order to ensure that all Config object are always in a proper state 

1020 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1021 some attributes are handled at allocation time rather than at 

1022 initialization. 

1023 

1024 This ensures that even if a derived `~lsst.pex.config.Config` class 

1025 implements ``__init__``, its author does not need to be concerned about 

1026 when or even the base ``Config.__init__`` should be called. 

1027 """ 

1028 name = kw.pop("__name", None) 

1029 at = kw.pop("__at", getCallStack()) 

1030 # remove __label and ignore it 

1031 kw.pop("__label", "default") 

1032 

1033 instance = object.__new__(cls) 

1034 instance._frozen = False 

1035 instance._name = name 

1036 instance._storage = {} 

1037 instance._history = {} 

1038 instance._imports = set() 

1039 # load up defaults 

1040 for field in instance._fields.values(): 

1041 instance._history[field.name] = [] 

1042 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1043 # set custom default-overrides 

1044 instance.setDefaults() 

1045 # set constructor overrides 

1046 instance.update(__at=at, **kw) 

1047 return instance 

1048 

1049 def __reduce__(self): 

1050 """Reduction for pickling (function with arguments to reproduce). 

1051 

1052 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1053 since it may contain lambdas (as the ``check`` elements) that cannot 

1054 be pickled. 

1055 """ 

1056 # The stream must be in characters to match the API but pickle 

1057 # requires bytes 

1058 stream = io.StringIO() 

1059 self.saveToStream(stream) 

1060 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1061 

1062 def setDefaults(self): 

1063 """Subclass hook for computing defaults. 

1064 

1065 Notes 

1066 ----- 

1067 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1068 rather than using the `~lsst.pex.config.Field` instances's defaults 

1069 should do so here. To correctly use inherited defaults, 

1070 implementations of ``setDefaults`` must call their base class's 

1071 ``setDefaults``. 

1072 """ 

1073 pass 

1074 

1075 def update(self, **kw): 

1076 """Update values of fields specified by the keyword arguments. 

1077 

1078 Parameters 

1079 ---------- 

1080 kw 

1081 Keywords are configuration field names. Values are configuration 

1082 field values. 

1083 

1084 Notes 

1085 ----- 

1086 The ``__at`` and ``__label`` keyword arguments are special internal 

1087 keywords. They are used to strip out any internal steps from the 

1088 history tracebacks of the config. Do not modify these keywords to 

1089 subvert a `~lsst.pex.config.Config` instance's history. 

1090 

1091 Examples 

1092 -------- 

1093 This is a config with three fields: 

1094 

1095 >>> from lsst.pex.config import Config, Field 

1096 >>> class DemoConfig(Config): 

1097 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1098 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1099 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1100 ... 

1101 >>> config = DemoConfig() 

1102 

1103 These are the default values of each field: 

1104 

1105 >>> for name, value in config.iteritems(): 

1106 ... print(f"{name}: {value}") 

1107 ... 

1108 fieldA: 42 

1109 fieldB: True 

1110 fieldC: 'Hello world' 

1111 

1112 Using this method to update ``fieldA`` and ``fieldC``: 

1113 

1114 >>> config.update(fieldA=13, fieldC='Updated!') 

1115 

1116 Now the values of each field are: 

1117 

1118 >>> for name, value in config.iteritems(): 

1119 ... print(f"{name}: {value}") 

1120 ... 

1121 fieldA: 13 

1122 fieldB: True 

1123 fieldC: 'Updated!' 

1124 """ 

1125 at = kw.pop("__at", getCallStack()) 

1126 label = kw.pop("__label", "update") 

1127 

1128 for name, value in kw.items(): 

1129 try: 

1130 field = self._fields[name] 

1131 field.__set__(self, value, at=at, label=label) 

1132 except KeyError: 

1133 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) 

1134 

1135 def load(self, filename, root="config"): 

1136 """Modify this config in place by executing the Python code in a 

1137 configuration file. 

1138 

1139 Parameters 

1140 ---------- 

1141 filename : `str` 

1142 Name of the configuration file. A configuration file is Python 

1143 module. 

1144 root : `str`, optional 

1145 Name of the variable in file that refers to the config being 

1146 overridden. 

1147 

1148 For example, the value of root is ``"config"`` and the file 

1149 contains:: 

1150 

1151 config.myField = 5 

1152 

1153 Then this config's field ``myField`` is set to ``5``. 

1154 

1155 See also 

1156 -------- 

1157 lsst.pex.config.Config.loadFromStream 

1158 lsst.pex.config.Config.loadFromString 

1159 lsst.pex.config.Config.save 

1160 lsst.pex.config.Config.saveToStream 

1161 lsst.pex.config.Config.saveToString 

1162 """ 

1163 with open(filename, "r") as f: 

1164 code = compile(f.read(), filename=filename, mode="exec") 

1165 self.loadFromString(code, root=root, filename=filename) 

1166 

1167 def loadFromStream(self, stream, root="config", filename=None): 

1168 """Modify this Config in place by executing the Python code in the 

1169 provided stream. 

1170 

1171 Parameters 

1172 ---------- 

1173 stream : file-like object, `str`, `bytes`, or compiled string 

1174 Stream containing configuration override code. If this is a 

1175 code object, it should be compiled with ``mode="exec"``. 

1176 root : `str`, optional 

1177 Name of the variable in file that refers to the config being 

1178 overridden. 

1179 

1180 For example, the value of root is ``"config"`` and the file 

1181 contains:: 

1182 

1183 config.myField = 5 

1184 

1185 Then this config's field ``myField`` is set to ``5``. 

1186 filename : `str`, optional 

1187 Name of the configuration file, or `None` if unknown or contained 

1188 in the stream. Used for error reporting. 

1189 

1190 Notes 

1191 ----- 

1192 For backwards compatibility reasons, this method accepts strings, bytes 

1193 and code objects as well as file-like objects. New code should use 

1194 `loadFromString` instead for most of these types. 

1195 

1196 See also 

1197 -------- 

1198 lsst.pex.config.Config.load 

1199 lsst.pex.config.Config.loadFromString 

1200 lsst.pex.config.Config.save 

1201 lsst.pex.config.Config.saveToStream 

1202 lsst.pex.config.Config.saveToString 

1203 """ 

1204 if hasattr(stream, "read"): 1204 ↛ 1205line 1204 didn't jump to line 1205, because the condition on line 1204 was never true

1205 if filename is None: 

1206 filename = getattr(stream, "name", "?") 

1207 code = compile(stream.read(), filename=filename, mode="exec") 

1208 else: 

1209 code = stream 

1210 self.loadFromString(code, root=root, filename=filename) 

1211 

1212 def loadFromString(self, code, root="config", filename=None): 

1213 """Modify this Config in place by executing the Python code in the 

1214 provided string. 

1215 

1216 Parameters 

1217 ---------- 

1218 code : `str`, `bytes`, or compiled string 

1219 Stream containing configuration override code. 

1220 root : `str`, optional 

1221 Name of the variable in file that refers to the config being 

1222 overridden. 

1223 

1224 For example, the value of root is ``"config"`` and the file 

1225 contains:: 

1226 

1227 config.myField = 5 

1228 

1229 Then this config's field ``myField`` is set to ``5``. 

1230 filename : `str`, optional 

1231 Name of the configuration file, or `None` if unknown or contained 

1232 in the stream. Used for error reporting. 

1233 

1234 See also 

1235 -------- 

1236 lsst.pex.config.Config.load 

1237 lsst.pex.config.Config.loadFromStream 

1238 lsst.pex.config.Config.save 

1239 lsst.pex.config.Config.saveToStream 

1240 lsst.pex.config.Config.saveToString 

1241 """ 

1242 if filename is None: 1242 ↛ 1246line 1242 didn't jump to line 1246, because the condition on line 1242 was never false

1243 # try to determine the file name; a compiled string 

1244 # has attribute "co_filename", 

1245 filename = getattr(code, "co_filename", "?") 

1246 with RecordingImporter() as importer: 

1247 globals = {"__file__": filename} 

1248 local = {root: self} 

1249 exec(code, globals, local) 

1250 

1251 self._imports.update(importer.getModules()) 

1252 

1253 def save(self, filename, root="config"): 

1254 """Save a Python script to the named file, which, when loaded, 

1255 reproduces this config. 

1256 

1257 Parameters 

1258 ---------- 

1259 filename : `str` 

1260 Desination filename of this configuration. 

1261 root : `str`, optional 

1262 Name to use for the root config variable. The same value must be 

1263 used when loading (see `lsst.pex.config.Config.load`). 

1264 

1265 See also 

1266 -------- 

1267 lsst.pex.config.Config.saveToStream 

1268 lsst.pex.config.Config.saveToString 

1269 lsst.pex.config.Config.load 

1270 lsst.pex.config.Config.loadFromStream 

1271 lsst.pex.config.Config.loadFromString 

1272 """ 

1273 d = os.path.dirname(filename) 

1274 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1275 self.saveToStream(outfile, root) 

1276 # tempfile is hardcoded to create files with mode '0600' 

1277 # for an explantion of these antics see: 

1278 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1279 umask = os.umask(0o077) 

1280 os.umask(umask) 

1281 os.chmod(outfile.name, (~umask & 0o666)) 

1282 # chmod before the move so we get quasi-atomic behavior if the 

1283 # source and dest. are on the same filesystem. 

1284 # os.rename may not work across filesystems 

1285 shutil.move(outfile.name, filename) 

1286 

1287 def saveToString(self, skipImports=False): 

1288 """Return the Python script form of this configuration as an executable 

1289 string. 

1290 

1291 Parameters 

1292 ---------- 

1293 skipImports : `bool`, optional 

1294 If `True` then do not include ``import`` statements in output, 

1295 this is to support human-oriented output from ``pipetask`` where 

1296 additional clutter is not useful. 

1297 

1298 Returns 

1299 ------- 

1300 code : `str` 

1301 A code string readable by `loadFromString`. 

1302 

1303 See also 

1304 -------- 

1305 lsst.pex.config.Config.save 

1306 lsst.pex.config.Config.saveToStream 

1307 lsst.pex.config.Config.load 

1308 lsst.pex.config.Config.loadFromStream 

1309 lsst.pex.config.Config.loadFromString 

1310 """ 

1311 buffer = io.StringIO() 

1312 self.saveToStream(buffer, skipImports=skipImports) 

1313 return buffer.getvalue() 

1314 

1315 def saveToStream(self, outfile, root="config", skipImports=False): 

1316 """Save a configuration file to a stream, which, when loaded, 

1317 reproduces this config. 

1318 

1319 Parameters 

1320 ---------- 

1321 outfile : file-like object 

1322 Destination file object write the config into. Accepts strings not 

1323 bytes. 

1324 root 

1325 Name to use for the root config variable. The same value must be 

1326 used when loading (see `lsst.pex.config.Config.load`). 

1327 skipImports : `bool`, optional 

1328 If `True` then do not include ``import`` statements in output, 

1329 this is to support human-oriented output from ``pipetask`` where 

1330 additional clutter is not useful. 

1331 

1332 See also 

1333 -------- 

1334 lsst.pex.config.Config.save 

1335 lsst.pex.config.Config.saveToString 

1336 lsst.pex.config.Config.load 

1337 lsst.pex.config.Config.loadFromStream 

1338 lsst.pex.config.Config.loadFromString 

1339 """ 

1340 tmp = self._name 

1341 self._rename(root) 

1342 try: 

1343 if not skipImports: 1343 ↛ 1357line 1343 didn't jump to line 1357, because the condition on line 1343 was never false

1344 self._collectImports() 

1345 # Remove self from the set, as it is handled explicitly below 

1346 self._imports.remove(self.__module__) 

1347 configType = type(self) 

1348 typeString = _typeStr(configType) 

1349 outfile.write(f"import {configType.__module__}\n") 

1350 outfile.write( 

1351 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1352 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1353 ) 

1354 for imp in sorted(self._imports): 1354 ↛ 1355line 1354 didn't jump to line 1355, because the loop on line 1354 never started

1355 if imp in sys.modules and sys.modules[imp] is not None: 

1356 outfile.write("import {}\n".format(imp)) 

1357 self._save(outfile) 

1358 finally: 

1359 self._rename(tmp) 

1360 

1361 def freeze(self): 

1362 """Make this config, and all subconfigs, read-only.""" 

1363 self._frozen = True 

1364 for field in self._fields.values(): 

1365 field.freeze(self) 

1366 

1367 def _save(self, outfile): 

1368 """Save this config to an open stream object. 

1369 

1370 Parameters 

1371 ---------- 

1372 outfile : file-like object 

1373 Destination file object write the config into. Accepts strings not 

1374 bytes. 

1375 """ 

1376 for field in self._fields.values(): 

1377 field.save(outfile, self) 

1378 

1379 def _collectImports(self): 

1380 """Adds module containing self to the list of things to import and 

1381 then loops over all the fields in the config calling a corresponding 

1382 collect method. The field method will call _collectImports on any 

1383 configs it may own and return the set of things to import. This 

1384 returned set will be merged with the set of imports for this config 

1385 class. 

1386 """ 

1387 self._imports.add(self.__module__) 

1388 for name, field in self._fields.items(): 

1389 field._collectImports(self, self._imports) 

1390 

1391 def toDict(self): 

1392 """Make a dictionary of field names and their values. 

1393 

1394 Returns 

1395 ------- 

1396 dict_ : `dict` 

1397 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1398 Values are `~lsst.pex.config.Field` values. 

1399 

1400 See also 

1401 -------- 

1402 lsst.pex.config.Field.toDict 

1403 

1404 Notes 

1405 ----- 

1406 This method uses the `~lsst.pex.config.Field.toDict` method of 

1407 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1408 implement a ``toDict`` method for *this* method to work. 

1409 """ 

1410 dict_ = {} 

1411 for name, field in self._fields.items(): 

1412 dict_[name] = field.toDict(self) 

1413 return dict_ 

1414 

1415 def names(self): 

1416 """Get all the field names in the config, recursively. 

1417 

1418 Returns 

1419 ------- 

1420 names : `list` of `str` 

1421 Field names. 

1422 """ 

1423 # 

1424 # Rather than sort out the recursion all over again use the 

1425 # pre-existing saveToStream() 

1426 # 

1427 with io.StringIO() as strFd: 

1428 self.saveToStream(strFd, "config") 

1429 contents = strFd.getvalue() 

1430 strFd.close() 

1431 # 

1432 # Pull the names out of the dumped config 

1433 # 

1434 keys = [] 

1435 for line in contents.split("\n"): 

1436 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1437 continue 

1438 

1439 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1440 if mat: 

1441 keys.append(mat.group(1)) 

1442 

1443 return keys 

1444 

1445 def _rename(self, name): 

1446 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1447 

1448 Parameters 

1449 ---------- 

1450 name : `str` 

1451 New name for this config in its parent `~lsst.pex.config.Config`. 

1452 

1453 Notes 

1454 ----- 

1455 This method uses the `~lsst.pex.config.Field.rename` method of 

1456 individual `lsst.pex.config.Field` instances. 

1457 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1458 method for *this* method to work. 

1459 

1460 See also 

1461 -------- 

1462 lsst.pex.config.Field.rename 

1463 """ 

1464 self._name = name 

1465 for field in self._fields.values(): 

1466 field.rename(self) 

1467 

1468 def validate(self): 

1469 """Validate the Config, raising an exception if invalid. 

1470 

1471 Raises 

1472 ------ 

1473 lsst.pex.config.FieldValidationError 

1474 Raised if verification fails. 

1475 

1476 Notes 

1477 ----- 

1478 The base class implementation performs type checks on all fields by 

1479 calling their `~lsst.pex.config.Field.validate` methods. 

1480 

1481 Complex single-field validation can be defined by deriving new Field 

1482 types. For convenience, some derived `lsst.pex.config.Field`-types 

1483 (`~lsst.pex.config.ConfigField` and 

1484 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config` 

1485 that handle recursing into subconfigs. 

1486 

1487 Inter-field relationships should only be checked in derived 

1488 `~lsst.pex.config.Config` classes after calling this method, and base 

1489 validation is complete. 

1490 """ 

1491 for field in self._fields.values(): 

1492 field.validate(self) 

1493 

1494 def formatHistory(self, name, **kwargs): 

1495 """Format a configuration field's history to a human-readable format. 

1496 

1497 Parameters 

1498 ---------- 

1499 name : `str` 

1500 Name of a `~lsst.pex.config.Field` in this config. 

1501 kwargs 

1502 Keyword arguments passed to `lsst.pex.config.history.format`. 

1503 

1504 Returns 

1505 ------- 

1506 history : `str` 

1507 A string containing the formatted history. 

1508 

1509 See also 

1510 -------- 

1511 lsst.pex.config.history.format 

1512 """ 

1513 import lsst.pex.config.history as pexHist 

1514 

1515 return pexHist.format(self, name, **kwargs) 

1516 

1517 history = property(lambda x: x._history) 1517 ↛ exitline 1517 didn't run the lambda on line 1517

1518 """Read-only history. 

1519 """ 

1520 

1521 def __setattr__(self, attr, value, at=None, label="assignment"): 

1522 """Set an attribute (such as a field's value). 

1523 

1524 Notes 

1525 ----- 

1526 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1527 locked such that no additional attributes nor properties may be added 

1528 to them dynamically. 

1529 

1530 Although this is not the standard Python behavior, it helps to protect 

1531 users from accidentally mispelling a field name, or trying to set a 

1532 non-existent field. 

1533 """ 

1534 if attr in self._fields: 

1535 if self._fields[attr].deprecated is not None: 1535 ↛ 1536line 1535 didn't jump to line 1536, because the condition on line 1535 was never true

1536 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1537 warnings.warn( 

1538 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1539 FutureWarning, 

1540 stacklevel=2, 

1541 ) 

1542 if at is None: 1542 ↛ 1545line 1542 didn't jump to line 1545, because the condition on line 1542 was never false

1543 at = getCallStack() 

1544 # This allows Field descriptors to work. 

1545 self._fields[attr].__set__(self, value, at=at, label=label) 

1546 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1546 ↛ 1548line 1546 didn't jump to line 1548, because the condition on line 1546 was never true

1547 # This allows properties and other non-Field descriptors to work. 

1548 return object.__setattr__(self, attr, value) 

1549 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1549 ↛ 1554line 1549 didn't jump to line 1554, because the condition on line 1549 was never false

1550 # This allows specific private attributes to work. 

1551 self.__dict__[attr] = value 

1552 else: 

1553 # We throw everything else. 

1554 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr)) 

1555 

1556 def __delattr__(self, attr, at=None, label="deletion"): 

1557 if attr in self._fields: 

1558 if at is None: 

1559 at = getCallStack() 

1560 self._fields[attr].__delete__(self, at=at, label=label) 

1561 else: 

1562 object.__delattr__(self, attr) 

1563 

1564 def __eq__(self, other): 

1565 if type(other) == type(self): 1565 ↛ 1566line 1565 didn't jump to line 1566, because the condition on line 1565 was never true

1566 for name in self._fields: 

1567 thisValue = getattr(self, name) 

1568 otherValue = getattr(other, name) 

1569 if isinstance(thisValue, float) and math.isnan(thisValue): 

1570 if not math.isnan(otherValue): 

1571 return False 

1572 elif thisValue != otherValue: 

1573 return False 

1574 return True 

1575 return False 

1576 

1577 def __ne__(self, other): 

1578 return not self.__eq__(other) 

1579 

1580 def __str__(self): 

1581 return str(self.toDict()) 

1582 

1583 def __repr__(self): 

1584 return "%s(%s)" % ( 

1585 _typeStr(self), 

1586 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None), 

1587 ) 

1588 

1589 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1590 """Compare this configuration to another `~lsst.pex.config.Config` for 

1591 equality. 

1592 

1593 Parameters 

1594 ---------- 

1595 other : `lsst.pex.config.Config` 

1596 Other `~lsst.pex.config.Config` object to compare against this 

1597 config. 

1598 shortcut : `bool`, optional 

1599 If `True`, return as soon as an inequality is found. Default is 

1600 `True`. 

1601 rtol : `float`, optional 

1602 Relative tolerance for floating point comparisons. 

1603 atol : `float`, optional 

1604 Absolute tolerance for floating point comparisons. 

1605 output : callable, optional 

1606 A callable that takes a string, used (possibly repeatedly) to 

1607 report inequalities. 

1608 

1609 Returns 

1610 ------- 

1611 isEqual : `bool` 

1612 `True` when the two `lsst.pex.config.Config` instances are equal. 

1613 `False` if there is an inequality. 

1614 

1615 See also 

1616 -------- 

1617 lsst.pex.config.compareConfigs 

1618 

1619 Notes 

1620 ----- 

1621 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1622 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1623 are not considered by this method. 

1624 

1625 Floating point comparisons are performed by `numpy.allclose`. 

1626 """ 

1627 name1 = self._name if self._name is not None else "config" 

1628 name2 = other._name if other._name is not None else "config" 

1629 name = getComparisonName(name1, name2) 

1630 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1631 

1632 @classmethod 

1633 def __init_subclass__(cls, **kwargs): 

1634 """Run initialization for every subclass. 

1635 

1636 Specifically registers the subclass with a YAML representer 

1637 and YAML constructor (if pyyaml is available) 

1638 """ 

1639 super().__init_subclass__(**kwargs) 

1640 

1641 if not yaml: 1641 ↛ 1642line 1641 didn't jump to line 1642, because the condition on line 1641 was never true

1642 return 

1643 

1644 yaml.add_representer(cls, _yaml_config_representer) 

1645 

1646 @classmethod 

1647 def _fromPython(cls, config_py): 

1648 """Instantiate a `Config`-subclass from serialized Python form. 

1649 

1650 Parameters 

1651 ---------- 

1652 config_py : `str` 

1653 A serialized form of the Config as created by 

1654 `Config.saveToStream`. 

1655 

1656 Returns 

1657 ------- 

1658 config : `Config` 

1659 Reconstructed `Config` instant. 

1660 """ 

1661 cls = _classFromPython(config_py) 

1662 return unreduceConfig(cls, config_py) 

1663 

1664 

1665def _classFromPython(config_py): 

1666 """Return the Config subclass required by this Config serialization. 

1667 

1668 Parameters 

1669 ---------- 

1670 config_py : `str` 

1671 A serialized form of the Config as created by 

1672 `Config.saveToStream`. 

1673 

1674 Returns 

1675 ------- 

1676 cls : `type` 

1677 The `Config` subclass associated with this config. 

1678 """ 

1679 # standard serialization has the form: 

1680 # import config.class 

1681 # assert type(config)==config.class.Config, ... 

1682 # We want to parse these two lines so we can get the class itself 

1683 

1684 # Do a single regex to avoid large string copies when splitting a 

1685 # large config into separate lines. 

1686 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1687 

1688 if not matches: 

1689 first_line, second_line, _ = config_py.split("\n", 2) 

1690 raise ValueError( 

1691 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}" 

1692 ) 

1693 

1694 module_name = matches.group(1) 

1695 module = importlib.import_module(module_name) 

1696 

1697 # Second line 

1698 full_name = matches.group(2) 

1699 

1700 # Remove the module name from the full name 

1701 if not full_name.startswith(module_name): 

1702 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1703 

1704 # if module name is a.b.c and full name is a.b.c.d.E then 

1705 # we need to remove a.b.c. and iterate over the remainder 

1706 # The +1 is for the extra dot after a.b.c 

1707 remainder = full_name[len(module_name) + 1 :] 

1708 components = remainder.split(".") 

1709 pytype = module 

1710 for component in components: 

1711 pytype = getattr(pytype, component) 

1712 return pytype 

1713 

1714 

1715def unreduceConfig(cls, stream): 

1716 """Create a `~lsst.pex.config.Config` from a stream. 

1717 

1718 Parameters 

1719 ---------- 

1720 cls : `lsst.pex.config.Config`-type 

1721 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1722 with configurations in the ``stream``. 

1723 stream : file-like object, `str`, or compiled string 

1724 Stream containing configuration override code. 

1725 

1726 Returns 

1727 ------- 

1728 config : `lsst.pex.config.Config` 

1729 Config instance. 

1730 

1731 See also 

1732 -------- 

1733 lsst.pex.config.Config.loadFromStream 

1734 """ 

1735 config = cls() 

1736 config.loadFromStream(stream) 

1737 return config