Coverage for python/lsst/pex/config/config.py: 58%

460 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-28 10:15 +0000

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from typing import Any, ForwardRef, Generic, Mapping, Optional, TypeVar, Union, cast, overload 

49 

50try: 

51 from types import GenericAlias 

52except ImportError: 

53 # cover python 3.8 usage 

54 GenericAlias = type(Mapping[int, int]) 

55 

56# if YAML is not available that's fine and we simply don't register 

57# the yaml representer since we know it won't be used. 

58try: 

59 import yaml 

60except ImportError: 

61 yaml = None 

62 

63from .callStack import getCallStack, getStackFrame 

64from .comparison import compareConfigs, compareScalars, getComparisonName 

65 

66if yaml: 66 ↛ 77line 66 didn't jump to line 77, because the condition on line 66 was never false

67 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

68 

69 try: 

70 # CLoader is not always available 

71 from yaml import CLoader 

72 

73 YamlLoaders += (CLoader,) 

74 except ImportError: 

75 pass 

76else: 

77 YamlLoaders = () 

78 doImport = None 

79 

80 

81if int(sys.version_info.minor) < 9: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true

82 genericAliasKwds = {"_root": True} 

83else: 

84 genericAliasKwds = {} 

85 

86 

87class _PexConfigGenericAlias(GenericAlias, **genericAliasKwds): 

88 """A Subclass of python's GenericAlias used in defining and instantiating 

89 Generics. 

90 

91 This class differs from `types.GenericAlias` in that it calls a method 

92 named _parseTypingArgs defined on Fields. This method gives Field and its 

93 subclasses an opportunity to transform type parameters into class key word 

94 arguments. Code authors do not need to implement any returns of this object 

95 directly, and instead only need implement _parseTypingArgs, if a Field 

96 subclass differs from the base class implementation. 

97 

98 This class is intended to be an implementation detail, returned from a 

99 Field's `__class_getitem__` method. 

100 """ 

101 

102 def __call__(self, *args: Any, **kwds: Any) -> Any: 

103 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

104 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

105 

106 

107FieldTypeVar = TypeVar("FieldTypeVar") 

108 

109 

110class UnexpectedProxyUsageError(TypeError): 

111 """Exception raised when a proxy class is used in a context that suggests 

112 it should have already been converted to the thing it proxies. 

113 """ 

114 

115 

116def _joinNamePath(prefix=None, name=None, index=None): 

117 """Generate nested configuration names.""" 

118 if not prefix and not name: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 raise ValueError("Invalid name: cannot be None") 

120 elif not name: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 name = prefix 

122 elif prefix and name: 122 ↛ 125line 122 didn't jump to line 125, because the condition on line 122 was never false

123 name = prefix + "." + name 

124 

125 if index is not None: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true

126 return "%s[%r]" % (name, index) 

127 else: 

128 return name 

129 

130 

131def _autocast(x, dtype): 

132 """Cast a value to a type, if appropriate. 

133 

134 Parameters 

135 ---------- 

136 x : object 

137 A value. 

138 dtype : tpye 

139 Data type, such as `float`, `int`, or `str`. 

140 

141 Returns 

142 ------- 

143 values : object 

144 If appropriate, the returned value is ``x`` cast to the given type 

145 ``dtype``. If the cast cannot be performed the original value of 

146 ``x`` is returned. 

147 """ 

148 if dtype == float and isinstance(x, int): 

149 return float(x) 

150 return x 

151 

152 

153def _typeStr(x): 

154 """Generate a fully-qualified type name. 

155 

156 Returns 

157 ------- 

158 `str` 

159 Fully-qualified type name. 

160 

161 Notes 

162 ----- 

163 This function is used primarily for writing config files to be executed 

164 later upon with the 'load' function. 

165 """ 

166 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

167 xtype = x 

168 else: 

169 xtype = type(x) 

170 if (sys.version_info.major <= 2 and xtype.__module__ == "__builtin__") or xtype.__module__ == "builtins": 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 return xtype.__name__ 

172 else: 

173 return "%s.%s" % (xtype.__module__, xtype.__name__) 

174 

175 

176if yaml: 176 ↛ 209line 176 didn't jump to line 209, because the condition on line 176 was never false

177 

178 def _yaml_config_representer(dumper, data): 

179 """Represent a Config object in a form suitable for YAML. 

180 

181 Stores the serialized stream as a scalar block string. 

182 """ 

183 stream = io.StringIO() 

184 data.saveToStream(stream) 

185 config_py = stream.getvalue() 

186 

187 # Strip multiple newlines from the end of the config 

188 # This simplifies the YAML to use | and not |+ 

189 config_py = config_py.rstrip() + "\n" 

190 

191 # Trailing spaces force pyyaml to use non-block form. 

192 # Remove the trailing spaces so it has no choice 

193 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

194 

195 # Store the Python as a simple scalar 

196 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

197 

198 def _yaml_config_constructor(loader, node): 

199 """Construct a config from YAML""" 

200 config_py = loader.construct_scalar(node) 

201 return Config._fromPython(config_py) 

202 

203 # Register a generic constructor for Config and all subclasses 

204 # Need to register for all the loaders we would like to use 

205 for loader in YamlLoaders: 

206 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

207 

208 

209class ConfigMeta(type): 

210 """A metaclass for `lsst.pex.config.Config`. 

211 

212 Notes 

213 ----- 

214 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

215 class attributes as a class attribute called ``_fields``, and adds 

216 the name of each field as an instance variable of the field itself (so you 

217 don't have to pass the name of the field to the field constructor). 

218 """ 

219 

220 def __init__(cls, name, bases, dict_): 

221 type.__init__(cls, name, bases, dict_) 

222 cls._fields = {} 

223 cls._source = getStackFrame() 

224 

225 def getFields(classtype): 

226 fields = {} 

227 bases = list(classtype.__bases__) 

228 bases.reverse() 

229 for b in bases: 

230 fields.update(getFields(b)) 

231 

232 for k, v in classtype.__dict__.items(): 

233 if isinstance(v, Field): 

234 fields[k] = v 

235 return fields 

236 

237 fields = getFields(cls) 

238 for k, v in fields.items(): 

239 setattr(cls, k, copy.deepcopy(v)) 

240 

241 def __setattr__(cls, name, value): 

242 if isinstance(value, Field): 

243 value.name = name 

244 cls._fields[name] = value 

245 type.__setattr__(cls, name, value) 

246 

247 

248class FieldValidationError(ValueError): 

249 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

250 particular ``~lsst.pex.config.Config``. 

251 

252 Parameters 

253 ---------- 

254 field : `lsst.pex.config.Field` 

255 The field that was not valid. 

256 config : `lsst.pex.config.Config` 

257 The config containing the invalid field. 

258 msg : `str` 

259 Text describing why the field was not valid. 

260 """ 

261 

262 def __init__(self, field, config, msg): 

263 self.fieldType = type(field) 

264 """Type of the `~lsst.pex.config.Field` that incurred the error. 

265 """ 

266 

267 self.fieldName = field.name 

268 """Name of the `~lsst.pex.config.Field` instance that incurred the 

269 error (`str`). 

270 

271 See also 

272 -------- 

273 lsst.pex.config.Field.name 

274 """ 

275 

276 self.fullname = _joinNamePath(config._name, field.name) 

277 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

278 (`str`). 

279 """ 

280 

281 self.history = config.history.setdefault(field.name, []) 

282 """Full history of all changes to the `~lsst.pex.config.Field` 

283 instance. 

284 """ 

285 

286 self.fieldSource = field.source 

287 """File and line number of the `~lsst.pex.config.Field` definition. 

288 """ 

289 

290 self.configSource = config._source 

291 error = ( 

292 "%s '%s' failed validation: %s\n" 

293 "For more information see the Field definition at:\n%s" 

294 " and the Config definition at:\n%s" 

295 % ( 

296 self.fieldType.__name__, 

297 self.fullname, 

298 msg, 

299 self.fieldSource.format(), 

300 self.configSource.format(), 

301 ) 

302 ) 

303 super().__init__(error) 

304 

305 

306class Field(Generic[FieldTypeVar]): 

307 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

308 `complex`, `bool`, and `str` data types. 

309 

310 Parameters 

311 ---------- 

312 doc : `str` 

313 A description of the field for users. 

314 dtype : type, optional 

315 The field's data type. ``Field`` only supports basic data types: 

316 `int`, `float`, `complex`, `bool`, and `str`. See 

317 `Field.supportedTypes`. Optional if supplied as a typing argument to 

318 the class. 

319 default : object, optional 

320 The field's default value. 

321 check : callable, optional 

322 A callable that is called with the field's value. This callable should 

323 return `False` if the value is invalid. More complex inter-field 

324 validation can be written as part of the 

325 `lsst.pex.config.Config.validate` method. 

326 optional : `bool`, optional 

327 This sets whether the field is considered optional, and therefore 

328 doesn't need to be set by the user. When `False`, 

329 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

330 deprecated : None or `str`, optional 

331 A description of why this Field is deprecated, including removal date. 

332 If not None, the string is appended to the docstring for this Field. 

333 

334 Raises 

335 ------ 

336 ValueError 

337 Raised when the ``dtype`` parameter is not one of the supported types 

338 (see `Field.supportedTypes`). 

339 

340 See also 

341 -------- 

342 ChoiceField 

343 ConfigChoiceField 

344 ConfigDictField 

345 ConfigField 

346 ConfigurableField 

347 DictField 

348 ListField 

349 RangeField 

350 RegistryField 

351 

352 Notes 

353 ----- 

354 ``Field`` instances (including those of any subclass of ``Field``) are used 

355 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

356 example, below). ``Field`` attributes work like the `property` attributes 

357 of classes that implement custom setters and getters. `Field` attributes 

358 belong to the class, but operate on the instance. Formally speaking, 

359 `Field` attributes are `descriptors 

360 <https://docs.python.org/3/howto/descriptor.html>`_. 

361 

362 When you access a `Field` attribute on a `Config` instance, you don't 

363 get the `Field` instance itself. Instead, you get the value of that field, 

364 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

365 container type (like a `lsst.pex.config.List`) depending on the field's 

366 type. See the example, below. 

367 

368 Fields can be annotated with a type similar to other python classes (python 

369 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

370 See the name field in the Config example below for an example of this. 

371 Unlike most other uses in python, this has an effect at type checking *and* 

372 runtime. If the type is specified with a class annotation, it will be used 

373 as the value of the ``dtype`` in the ``Field`` and there is no need to 

374 specify it as an argument during instantiation. 

375 

376 There are Some notes on dtype through type annotation syntax. Type 

377 annotation syntax supports supplying the argument as a string of a type 

378 name. i.e. "float", but this cannot be used to resolve circular references. 

379 Type annotation syntax can be used on an identifier in addition to Class 

380 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

381 ``someField = Field[str](doc="some doc"). However, this syntax is only 

382 useful for annotating the type of the identifier (i.e. variable in previous 

383 example) and does nothing for assigning the dtype of the ``Field``. 

384 

385 

386 Examples 

387 -------- 

388 Instances of ``Field`` should be used as class attributes of 

389 `lsst.pex.config.Config` subclasses: 

390 

391 >>> from lsst.pex.config import Config, Field 

392 >>> class Example(Config): 

393 ... myInt = Field("An integer field.", int, default=0) 

394 ... name = Field[str](doc="A string Field") 

395 ... 

396 >>> print(config.myInt) 

397 0 

398 >>> config.myInt = 5 

399 >>> print(config.myInt) 

400 5 

401 """ 

402 

403 name: str 

404 """Identifier (variable name) used to refer to a Field within a Config 

405 Class. 

406 """ 

407 

408 supportedTypes = set((str, bool, float, int, complex)) 

409 """Supported data types for field values (`set` of types). 

410 """ 

411 

412 @staticmethod 

413 def _parseTypingArgs( 

414 params: Union[tuple[type, ...], tuple[str, ...]], kwds: Mapping[str, Any] 

415 ) -> Mapping[str, Any]: 

416 """Parses type annotations into keyword constructor arguments. 

417 

418 This is a special private method that interprets type arguments (i.e. 

419 Field[str]) into keyword arguments to be passed on to the constructor. 

420 

421 Subclasses of Field can implement this method to customize how they 

422 handle turning type parameters into keyword arguments (see DictField 

423 for an example) 

424 

425 Parameters 

426 ---------- 

427 params : `tuple` of `type` or `tuple` of str 

428 Parameters passed to the type annotation. These will either be 

429 types or strings. Strings are to interpreted as forward references 

430 and will be treated as such. 

431 kwds : `MutableMapping` with keys of `str` and values of `Any` 

432 These are the user supplied keywords that are to be passed to the 

433 Field constructor. 

434 

435 Returns 

436 ------- 

437 kwds : `MutableMapping` with keys of `str` and values of `Any` 

438 The mapping of keywords that will be passed onto the constructor 

439 of the Field. Should be filled in with any information gleaned 

440 from the input parameters. 

441 

442 Raises 

443 ------ 

444 ValueError : 

445 Raised if params is of incorrect length. 

446 Raised if a forward reference could not be resolved 

447 Raised if there is a conflict between params and values in kwds 

448 """ 

449 if len(params) > 1: 

450 raise ValueError("Only single type parameters are supported") 

451 unpackedParams = params[0] 

452 if isinstance(unpackedParams, str): 

453 _typ = ForwardRef(unpackedParams) 

454 # type ignore below because typeshed seems to be wrong. It 

455 # indicates there are only 2 args, as it was in python 3.8, but 

456 # 3.9+ takes 3 args. Attempt in old style and new style to 

457 # work with both. 

458 try: 

459 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

460 except TypeError: 

461 # python 3.8 path 

462 result = _typ._evaluate(globals(), locals()) 

463 if result is None: 

464 raise ValueError("Could not deduce type from input") 

465 unpackedParams = cast(type, result) 

466 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

467 raise ValueError("Conflicting definition for dtype") 

468 elif "dtype" not in kwds: 

469 kwds = {**kwds, **{"dtype": unpackedParams}} 

470 return kwds 

471 

472 def __class_getitem__(cls, params: Union[tuple[type, ...], type, ForwardRef]): 

473 return _PexConfigGenericAlias(cls, params) 

474 

475 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

476 if dtype is None: 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 raise ValueError( 

478 "dtype must either be supplied as an argument or as a type argument to the class" 

479 ) 

480 if dtype not in self.supportedTypes: 480 ↛ 481line 480 didn't jump to line 481, because the condition on line 480 was never true

481 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

482 

483 source = getStackFrame() 

484 self._setup( 

485 doc=doc, 

486 dtype=dtype, 

487 default=default, 

488 check=check, 

489 optional=optional, 

490 source=source, 

491 deprecated=deprecated, 

492 ) 

493 

494 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

495 """Set attributes, usually during initialization.""" 

496 self.dtype = dtype 

497 """Data type for the field. 

498 """ 

499 

500 if not doc: 500 ↛ 501line 500 didn't jump to line 501, because the condition on line 500 was never true

501 raise ValueError("Docstring is empty.") 

502 

503 # append the deprecation message to the docstring. 

504 if deprecated is not None: 

505 doc = f"{doc} Deprecated: {deprecated}" 

506 self.doc = doc 

507 """A description of the field (`str`). 

508 """ 

509 

510 self.deprecated = deprecated 

511 """If not None, a description of why this field is deprecated (`str`). 

512 """ 

513 

514 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

515 if optional or default is not None: 

516 self.__doc__ += f", default ``{default!r}``" 

517 self.__doc__ += ")" 

518 

519 self.default = default 

520 """Default value for this field. 

521 """ 

522 

523 self.check = check 

524 """A user-defined function that validates the value of the field. 

525 """ 

526 

527 self.optional = optional 

528 """Flag that determines if the field is required to be set (`bool`). 

529 

530 When `False`, `lsst.pex.config.Config.validate` will fail if the 

531 field's value is `None`. 

532 """ 

533 

534 self.source = source 

535 """The stack frame where this field is defined (`list` of 

536 `lsst.pex.config.callStack.StackFrame`). 

537 """ 

538 

539 def rename(self, instance): 

540 """Rename the field in a `~lsst.pex.config.Config` (for internal use 

541 only). 

542 

543 Parameters 

544 ---------- 

545 instance : `lsst.pex.config.Config` 

546 The config instance that contains this field. 

547 

548 Notes 

549 ----- 

550 This method is invoked by the `lsst.pex.config.Config` object that 

551 contains this field and should not be called directly. 

552 

553 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

554 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should 

555 rename each subconfig with the full field name as generated by 

556 `lsst.pex.config.config._joinNamePath`. 

557 """ 

558 pass 

559 

560 def validate(self, instance): 

561 """Validate the field (for internal use only). 

562 

563 Parameters 

564 ---------- 

565 instance : `lsst.pex.config.Config` 

566 The config instance that contains this field. 

567 

568 Raises 

569 ------ 

570 lsst.pex.config.FieldValidationError 

571 Raised if verification fails. 

572 

573 Notes 

574 ----- 

575 This method provides basic validation: 

576 

577 - Ensures that the value is not `None` if the field is not optional. 

578 - Ensures type correctness. 

579 - Ensures that the user-provided ``check`` function is valid. 

580 

581 Most `~lsst.pex.config.Field` subclasses should call 

582 `lsst.pex.config.field.Field.validate` if they re-implement 

583 `~lsst.pex.config.field.Field.validate`. 

584 """ 

585 value = self.__get__(instance) 

586 if not self.optional and value is None: 

587 raise FieldValidationError(self, instance, "Required value cannot be None") 

588 

589 def freeze(self, instance): 

590 """Make this field read-only (for internal use only). 

591 

592 Parameters 

593 ---------- 

594 instance : `lsst.pex.config.Config` 

595 The config instance that contains this field. 

596 

597 Notes 

598 ----- 

599 Freezing is only relevant for fields that hold subconfigs. Fields which 

600 hold subconfigs should freeze each subconfig. 

601 

602 **Subclasses should implement this method.** 

603 """ 

604 pass 

605 

606 def _validateValue(self, value): 

607 """Validate a value. 

608 

609 Parameters 

610 ---------- 

611 value : object 

612 The value being validated. 

613 

614 Raises 

615 ------ 

616 TypeError 

617 Raised if the value's type is incompatible with the field's 

618 ``dtype``. 

619 ValueError 

620 Raised if the value is rejected by the ``check`` method. 

621 """ 

622 if value is None: 622 ↛ 623line 622 didn't jump to line 623, because the condition on line 622 was never true

623 return 

624 

625 if not isinstance(value, self.dtype): 625 ↛ 626line 625 didn't jump to line 626, because the condition on line 625 was never true

626 msg = "Value %s is of incorrect type %s. Expected type %s" % ( 

627 value, 

628 _typeStr(value), 

629 _typeStr(self.dtype), 

630 ) 

631 raise TypeError(msg) 

632 if self.check is not None and not self.check(value): 632 ↛ 633line 632 didn't jump to line 633, because the condition on line 632 was never true

633 msg = "Value %s is not a valid value" % str(value) 

634 raise ValueError(msg) 

635 

636 def _collectImports(self, instance, imports): 

637 """This function should call the _collectImports method on all config 

638 objects the field may own, and union them with the supplied imports 

639 set. 

640 

641 Parameters 

642 ---------- 

643 instance : instance or subclass of `lsst.pex.config.Config` 

644 A config object that has this field defined on it 

645 imports : `set` 

646 Set of python modules that need imported after persistence 

647 """ 

648 pass 

649 

650 def save(self, outfile, instance): 

651 """Save this field to a file (for internal use only). 

652 

653 Parameters 

654 ---------- 

655 outfile : file-like object 

656 A writeable field handle. 

657 instance : `Config` 

658 The `Config` instance that contains this field. 

659 

660 Notes 

661 ----- 

662 This method is invoked by the `~lsst.pex.config.Config` object that 

663 contains this field and should not be called directly. 

664 

665 The output consists of the documentation string 

666 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

667 line is formatted as an assignment: ``{fullname}={value}``. 

668 

669 This output can be executed with Python. 

670 """ 

671 value = self.__get__(instance) 

672 fullname = _joinNamePath(instance._name, self.name) 

673 

674 if self.deprecated and value == self.default: 674 ↛ 675line 674 didn't jump to line 675, because the condition on line 674 was never true

675 return 

676 

677 # write full documentation string as comment lines 

678 # (i.e. first character is #) 

679 doc = "# " + str(self.doc).replace("\n", "\n# ") 

680 if isinstance(value, float) and not math.isfinite(value): 680 ↛ 682line 680 didn't jump to line 682, because the condition on line 680 was never true

681 # non-finite numbers need special care 

682 outfile.write("{}\n{}=float('{!r}')\n\n".format(doc, fullname, value)) 

683 else: 

684 outfile.write("{}\n{}={!r}\n\n".format(doc, fullname, value)) 

685 

686 def toDict(self, instance): 

687 """Convert the field value so that it can be set as the value of an 

688 item in a `dict` (for internal use only). 

689 

690 Parameters 

691 ---------- 

692 instance : `Config` 

693 The `Config` that contains this field. 

694 

695 Returns 

696 ------- 

697 value : object 

698 The field's value. See *Notes*. 

699 

700 Notes 

701 ----- 

702 This method invoked by the owning `~lsst.pex.config.Config` object and 

703 should not be called directly. 

704 

705 Simple values are passed through. Complex data structures must be 

706 manipulated. For example, a `~lsst.pex.config.Field` holding a 

707 subconfig should, instead of the subconfig object, return a `dict` 

708 where the keys are the field names in the subconfig, and the values are 

709 the field values in the subconfig. 

710 """ 

711 return self.__get__(instance) 

712 

713 @overload 

714 def __get__( 

715 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

716 ) -> "Field[FieldTypeVar]": 

717 ... 

718 

719 @overload 

720 def __get__( 

721 self, instance: "Config", owner: Any = None, at: Any = None, label: str = "default" 

722 ) -> FieldTypeVar: 

723 ... 

724 

725 def __get__(self, instance, owner=None, at=None, label="default"): 

726 """Define how attribute access should occur on the Config instance 

727 This is invoked by the owning config object and should not be called 

728 directly 

729 

730 When the field attribute is accessed on a Config class object, it 

731 returns the field object itself in order to allow inspection of 

732 Config classes. 

733 

734 When the field attribute is access on a config instance, the actual 

735 value described by the field (and held by the Config instance) is 

736 returned. 

737 """ 

738 if instance is None: 738 ↛ 739line 738 didn't jump to line 739, because the condition on line 738 was never true

739 return self 

740 else: 

741 # try statements are almost free in python if they succeed 

742 try: 

743 return instance._storage[self.name] 

744 except AttributeError: 

745 if not isinstance(instance, Config): 

746 return self 

747 else: 

748 raise AttributeError( 

749 f"Config {instance} is missing _storage attribute, likely incorrectly initialized" 

750 ) 

751 

752 def __set__( 

753 self, instance: "Config", value: Optional[FieldTypeVar], at: Any = None, label: str = "assignment" 

754 ) -> None: 

755 """Set an attribute on the config instance. 

756 

757 Parameters 

758 ---------- 

759 instance : `lsst.pex.config.Config` 

760 The config instance that contains this field. 

761 value : obj 

762 Value to set on this field. 

763 at : `list` of `lsst.pex.config.callStack.StackFrame` 

764 The call stack (created by 

765 `lsst.pex.config.callStack.getCallStack`). 

766 label : `str`, optional 

767 Event label for the history. 

768 

769 Notes 

770 ----- 

771 This method is invoked by the owning `lsst.pex.config.Config` object 

772 and should not be called directly. 

773 

774 Derived `~lsst.pex.config.Field` classes may need to override the 

775 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

776 should follow the following rules: 

777 

778 - Do not allow modification of frozen configs. 

779 - Validate the new value **before** modifying the field. Except if the 

780 new value is `None`. `None` is special and no attempt should be made 

781 to validate it until `lsst.pex.config.Config.validate` is called. 

782 - Do not modify the `~lsst.pex.config.Config` instance to contain 

783 invalid values. 

784 - If the field is modified, update the history of the 

785 `lsst.pex.config.field.Field` to reflect the changes. 

786 

787 In order to decrease the need to implement this method in derived 

788 `~lsst.pex.config.Field` types, value validation is performed in the 

789 `lsst.pex.config.Field._validateValue`. If only the validation step 

790 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

791 implement `lsst.pex.config.Field._validateValue` than to reimplement 

792 ``__set__``. More complicated behavior, however, may require 

793 reimplementation. 

794 """ 

795 if instance._frozen: 795 ↛ 796line 795 didn't jump to line 796, because the condition on line 795 was never true

796 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

797 

798 history = instance._history.setdefault(self.name, []) 

799 if value is not None: 799 ↛ 806line 799 didn't jump to line 806, because the condition on line 799 was never false

800 value = _autocast(value, self.dtype) 

801 try: 

802 self._validateValue(value) 

803 except BaseException as e: 

804 raise FieldValidationError(self, instance, str(e)) 

805 

806 instance._storage[self.name] = value 

807 if at is None: 807 ↛ 808line 807 didn't jump to line 808, because the condition on line 807 was never true

808 at = getCallStack() 

809 history.append((value, at, label)) 

810 

811 def __delete__(self, instance, at=None, label="deletion"): 

812 """Delete an attribute from a `lsst.pex.config.Config` instance. 

813 

814 Parameters 

815 ---------- 

816 instance : `lsst.pex.config.Config` 

817 The config instance that contains this field. 

818 at : `list` of `lsst.pex.config.callStack.StackFrame` 

819 The call stack (created by 

820 `lsst.pex.config.callStack.getCallStack`). 

821 label : `str`, optional 

822 Event label for the history. 

823 

824 Notes 

825 ----- 

826 This is invoked by the owning `~lsst.pex.config.Config` object and 

827 should not be called directly. 

828 """ 

829 if at is None: 

830 at = getCallStack() 

831 self.__set__(instance, None, at=at, label=label) 

832 

833 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

834 """Compare a field (named `Field.name`) in two 

835 `~lsst.pex.config.Config` instances for equality. 

836 

837 Parameters 

838 ---------- 

839 instance1 : `lsst.pex.config.Config` 

840 Left-hand side `Config` instance to compare. 

841 instance2 : `lsst.pex.config.Config` 

842 Right-hand side `Config` instance to compare. 

843 shortcut : `bool`, optional 

844 **Unused.** 

845 rtol : `float`, optional 

846 Relative tolerance for floating point comparisons. 

847 atol : `float`, optional 

848 Absolute tolerance for floating point comparisons. 

849 output : callable, optional 

850 A callable that takes a string, used (possibly repeatedly) to 

851 report inequalities. 

852 

853 Notes 

854 ----- 

855 This method must be overridden by more complex `Field` subclasses. 

856 

857 See also 

858 -------- 

859 lsst.pex.config.compareScalars 

860 """ 

861 v1 = getattr(instance1, self.name) 

862 v2 = getattr(instance2, self.name) 

863 name = getComparisonName( 

864 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

865 ) 

866 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

867 

868 

869class RecordingImporter: 

870 """Importer (for `sys.meta_path`) that records which modules are being 

871 imported. 

872 

873 *This class does not do any importing itself.* 

874 

875 Examples 

876 -------- 

877 Use this class as a context manager to ensure it is properly uninstalled 

878 when done: 

879 

880 >>> with RecordingImporter() as importer: 

881 ... # import stuff 

882 ... import numpy as np 

883 ... print("Imported: " + importer.getModules()) 

884 """ 

885 

886 def __init__(self): 

887 self._modules = set() 

888 

889 def __enter__(self): 

890 self.origMetaPath = sys.meta_path 

891 sys.meta_path = [self] + sys.meta_path # type: ignore 

892 return self 

893 

894 def __exit__(self, *args): 

895 self.uninstall() 

896 return False # Don't suppress exceptions 

897 

898 def uninstall(self): 

899 """Uninstall the importer.""" 

900 sys.meta_path = self.origMetaPath 

901 

902 def find_module(self, fullname, path=None): 

903 """Called as part of the ``import`` chain of events.""" 

904 self._modules.add(fullname) 

905 # Return None because we don't do any importing. 

906 return None 

907 

908 def getModules(self): 

909 """Get the set of modules that were imported. 

910 

911 Returns 

912 ------- 

913 modules : `set` of `str` 

914 Set of imported module names. 

915 """ 

916 return self._modules 

917 

918 

919# type ignore because type checker thinks ConfigMeta is Generic when it is not 

920class Config(metaclass=ConfigMeta): # type: ignore 

921 """Base class for configuration (*config*) objects. 

922 

923 Notes 

924 ----- 

925 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

926 instances as class attributes. These are used to define most of the base 

927 class behavior. 

928 

929 ``Config`` implements a mapping API that provides many `dict`-like methods, 

930 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and 

931 `itervalues`. ``Config`` instances also support the ``in`` operator to 

932 test if a field is in the config. Unlike a `dict`, ``Config`` classes are 

933 not subscriptable. Instead, access individual fields as attributes of the 

934 configuration instance. 

935 

936 Examples 

937 -------- 

938 Config classes are subclasses of ``Config`` that have 

939 `~lsst.pex.config.Field` instances (or instances of 

940 `~lsst.pex.config.Field` subclasses) as class attributes: 

941 

942 >>> from lsst.pex.config import Config, Field, ListField 

943 >>> class DemoConfig(Config): 

944 ... intField = Field(doc="An integer field", dtype=int, default=42) 

945 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

946 ... default=['coffee', 'green tea', 'water']) 

947 ... 

948 >>> config = DemoConfig() 

949 

950 Configs support many `dict`-like APIs: 

951 

952 >>> config.keys() 

953 ['intField', 'listField'] 

954 >>> 'intField' in config 

955 True 

956 

957 Individual fields can be accessed as attributes of the configuration: 

958 

959 >>> config.intField 

960 42 

961 >>> config.listField.append('earl grey tea') 

962 >>> print(config.listField) 

963 ['coffee', 'green tea', 'water', 'earl grey tea'] 

964 """ 

965 

966 _storage: dict[str, Any] 

967 _fields: dict[str, Field] 

968 _history: dict[str, list[Any]] 

969 _imports: set[Any] 

970 

971 def __iter__(self): 

972 """Iterate over fields.""" 

973 return self._fields.__iter__() 

974 

975 def keys(self): 

976 """Get field names. 

977 

978 Returns 

979 ------- 

980 names : `dict_keys` 

981 List of `lsst.pex.config.Field` names. 

982 

983 See also 

984 -------- 

985 lsst.pex.config.Config.iterkeys 

986 """ 

987 return self._storage.keys() 

988 

989 def values(self): 

990 """Get field values. 

991 

992 Returns 

993 ------- 

994 values : `dict_values` 

995 Iterator of field values. 

996 """ 

997 return self._storage.values() 

998 

999 def items(self): 

1000 """Get configurations as ``(field name, field value)`` pairs. 

1001 

1002 Returns 

1003 ------- 

1004 items : `dict_items` 

1005 Iterator of tuples for each configuration. Tuple items are: 

1006 

1007 0. Field name. 

1008 1. Field value. 

1009 """ 

1010 return self._storage.items() 

1011 

1012 def __contains__(self, name): 

1013 """!Return True if the specified field exists in this config 

1014 

1015 @param[in] name field name to test for 

1016 """ 

1017 return self._storage.__contains__(name) 

1018 

1019 def __new__(cls, *args, **kw): 

1020 """Allocate a new `lsst.pex.config.Config` object. 

1021 

1022 In order to ensure that all Config object are always in a proper state 

1023 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1024 some attributes are handled at allocation time rather than at 

1025 initialization. 

1026 

1027 This ensures that even if a derived `~lsst.pex.config.Config` class 

1028 implements ``__init__``, its author does not need to be concerned about 

1029 when or even the base ``Config.__init__`` should be called. 

1030 """ 

1031 name = kw.pop("__name", None) 

1032 at = kw.pop("__at", getCallStack()) 

1033 # remove __label and ignore it 

1034 kw.pop("__label", "default") 

1035 

1036 instance = object.__new__(cls) 

1037 instance._frozen = False 

1038 instance._name = name 

1039 instance._storage = {} 

1040 instance._history = {} 

1041 instance._imports = set() 

1042 # load up defaults 

1043 for field in instance._fields.values(): 

1044 instance._history[field.name] = [] 

1045 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1046 # set custom default-overrides 

1047 instance.setDefaults() 

1048 # set constructor overrides 

1049 instance.update(__at=at, **kw) 

1050 return instance 

1051 

1052 def __reduce__(self): 

1053 """Reduction for pickling (function with arguments to reproduce). 

1054 

1055 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1056 since it may contain lambdas (as the ``check`` elements) that cannot 

1057 be pickled. 

1058 """ 

1059 # The stream must be in characters to match the API but pickle 

1060 # requires bytes 

1061 stream = io.StringIO() 

1062 self.saveToStream(stream) 

1063 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1064 

1065 def setDefaults(self): 

1066 """Subclass hook for computing defaults. 

1067 

1068 Notes 

1069 ----- 

1070 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1071 rather than using the `~lsst.pex.config.Field` instances's defaults 

1072 should do so here. To correctly use inherited defaults, 

1073 implementations of ``setDefaults`` must call their base class's 

1074 ``setDefaults``. 

1075 """ 

1076 pass 

1077 

1078 def update(self, **kw): 

1079 """Update values of fields specified by the keyword arguments. 

1080 

1081 Parameters 

1082 ---------- 

1083 kw 

1084 Keywords are configuration field names. Values are configuration 

1085 field values. 

1086 

1087 Notes 

1088 ----- 

1089 The ``__at`` and ``__label`` keyword arguments are special internal 

1090 keywords. They are used to strip out any internal steps from the 

1091 history tracebacks of the config. Do not modify these keywords to 

1092 subvert a `~lsst.pex.config.Config` instance's history. 

1093 

1094 Examples 

1095 -------- 

1096 This is a config with three fields: 

1097 

1098 >>> from lsst.pex.config import Config, Field 

1099 >>> class DemoConfig(Config): 

1100 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1101 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1102 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1103 ... 

1104 >>> config = DemoConfig() 

1105 

1106 These are the default values of each field: 

1107 

1108 >>> for name, value in config.iteritems(): 

1109 ... print(f"{name}: {value}") 

1110 ... 

1111 fieldA: 42 

1112 fieldB: True 

1113 fieldC: 'Hello world' 

1114 

1115 Using this method to update ``fieldA`` and ``fieldC``: 

1116 

1117 >>> config.update(fieldA=13, fieldC='Updated!') 

1118 

1119 Now the values of each field are: 

1120 

1121 >>> for name, value in config.iteritems(): 

1122 ... print(f"{name}: {value}") 

1123 ... 

1124 fieldA: 13 

1125 fieldB: True 

1126 fieldC: 'Updated!' 

1127 """ 

1128 at = kw.pop("__at", getCallStack()) 

1129 label = kw.pop("__label", "update") 

1130 

1131 for name, value in kw.items(): 

1132 try: 

1133 field = self._fields[name] 

1134 field.__set__(self, value, at=at, label=label) 

1135 except KeyError: 

1136 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) 

1137 

1138 def load(self, filename, root="config"): 

1139 """Modify this config in place by executing the Python code in a 

1140 configuration file. 

1141 

1142 Parameters 

1143 ---------- 

1144 filename : `str` 

1145 Name of the configuration file. A configuration file is Python 

1146 module. 

1147 root : `str`, optional 

1148 Name of the variable in file that refers to the config being 

1149 overridden. 

1150 

1151 For example, the value of root is ``"config"`` and the file 

1152 contains:: 

1153 

1154 config.myField = 5 

1155 

1156 Then this config's field ``myField`` is set to ``5``. 

1157 

1158 See also 

1159 -------- 

1160 lsst.pex.config.Config.loadFromStream 

1161 lsst.pex.config.Config.loadFromString 

1162 lsst.pex.config.Config.save 

1163 lsst.pex.config.Config.saveToStream 

1164 lsst.pex.config.Config.saveToString 

1165 """ 

1166 with open(filename, "r") as f: 

1167 code = compile(f.read(), filename=filename, mode="exec") 

1168 self.loadFromString(code, root=root, filename=filename) 

1169 

1170 def loadFromStream(self, stream, root="config", filename=None): 

1171 """Modify this Config in place by executing the Python code in the 

1172 provided stream. 

1173 

1174 Parameters 

1175 ---------- 

1176 stream : file-like object, `str`, `bytes`, or compiled string 

1177 Stream containing configuration override code. If this is a 

1178 code object, it should be compiled with ``mode="exec"``. 

1179 root : `str`, optional 

1180 Name of the variable in file that refers to the config being 

1181 overridden. 

1182 

1183 For example, the value of root is ``"config"`` and the file 

1184 contains:: 

1185 

1186 config.myField = 5 

1187 

1188 Then this config's field ``myField`` is set to ``5``. 

1189 filename : `str`, optional 

1190 Name of the configuration file, or `None` if unknown or contained 

1191 in the stream. Used for error reporting. 

1192 

1193 Notes 

1194 ----- 

1195 For backwards compatibility reasons, this method accepts strings, bytes 

1196 and code objects as well as file-like objects. New code should use 

1197 `loadFromString` instead for most of these types. 

1198 

1199 See also 

1200 -------- 

1201 lsst.pex.config.Config.load 

1202 lsst.pex.config.Config.loadFromString 

1203 lsst.pex.config.Config.save 

1204 lsst.pex.config.Config.saveToStream 

1205 lsst.pex.config.Config.saveToString 

1206 """ 

1207 if hasattr(stream, "read"): 1207 ↛ 1208line 1207 didn't jump to line 1208, because the condition on line 1207 was never true

1208 if filename is None: 

1209 filename = getattr(stream, "name", "?") 

1210 code = compile(stream.read(), filename=filename, mode="exec") 

1211 else: 

1212 code = stream 

1213 self.loadFromString(code, root=root, filename=filename) 

1214 

1215 def loadFromString(self, code, root="config", filename=None): 

1216 """Modify this Config in place by executing the Python code in the 

1217 provided string. 

1218 

1219 Parameters 

1220 ---------- 

1221 code : `str`, `bytes`, or compiled string 

1222 Stream containing configuration override code. 

1223 root : `str`, optional 

1224 Name of the variable in file that refers to the config being 

1225 overridden. 

1226 

1227 For example, the value of root is ``"config"`` and the file 

1228 contains:: 

1229 

1230 config.myField = 5 

1231 

1232 Then this config's field ``myField`` is set to ``5``. 

1233 filename : `str`, optional 

1234 Name of the configuration file, or `None` if unknown or contained 

1235 in the stream. Used for error reporting. 

1236 

1237 See also 

1238 -------- 

1239 lsst.pex.config.Config.load 

1240 lsst.pex.config.Config.loadFromStream 

1241 lsst.pex.config.Config.save 

1242 lsst.pex.config.Config.saveToStream 

1243 lsst.pex.config.Config.saveToString 

1244 """ 

1245 if filename is None: 1245 ↛ 1249line 1245 didn't jump to line 1249, because the condition on line 1245 was never false

1246 # try to determine the file name; a compiled string 

1247 # has attribute "co_filename", 

1248 filename = getattr(code, "co_filename", "?") 

1249 with RecordingImporter() as importer: 

1250 globals = {"__file__": filename} 

1251 local = {root: self} 

1252 exec(code, globals, local) 

1253 

1254 self._imports.update(importer.getModules()) 

1255 

1256 def save(self, filename, root="config"): 

1257 """Save a Python script to the named file, which, when loaded, 

1258 reproduces this config. 

1259 

1260 Parameters 

1261 ---------- 

1262 filename : `str` 

1263 Desination filename of this configuration. 

1264 root : `str`, optional 

1265 Name to use for the root config variable. The same value must be 

1266 used when loading (see `lsst.pex.config.Config.load`). 

1267 

1268 See also 

1269 -------- 

1270 lsst.pex.config.Config.saveToStream 

1271 lsst.pex.config.Config.saveToString 

1272 lsst.pex.config.Config.load 

1273 lsst.pex.config.Config.loadFromStream 

1274 lsst.pex.config.Config.loadFromString 

1275 """ 

1276 d = os.path.dirname(filename) 

1277 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1278 self.saveToStream(outfile, root) 

1279 # tempfile is hardcoded to create files with mode '0600' 

1280 # for an explantion of these antics see: 

1281 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1282 umask = os.umask(0o077) 

1283 os.umask(umask) 

1284 os.chmod(outfile.name, (~umask & 0o666)) 

1285 # chmod before the move so we get quasi-atomic behavior if the 

1286 # source and dest. are on the same filesystem. 

1287 # os.rename may not work across filesystems 

1288 shutil.move(outfile.name, filename) 

1289 

1290 def saveToString(self, skipImports=False): 

1291 """Return the Python script form of this configuration as an executable 

1292 string. 

1293 

1294 Parameters 

1295 ---------- 

1296 skipImports : `bool`, optional 

1297 If `True` then do not include ``import`` statements in output, 

1298 this is to support human-oriented output from ``pipetask`` where 

1299 additional clutter is not useful. 

1300 

1301 Returns 

1302 ------- 

1303 code : `str` 

1304 A code string readable by `loadFromString`. 

1305 

1306 See also 

1307 -------- 

1308 lsst.pex.config.Config.save 

1309 lsst.pex.config.Config.saveToStream 

1310 lsst.pex.config.Config.load 

1311 lsst.pex.config.Config.loadFromStream 

1312 lsst.pex.config.Config.loadFromString 

1313 """ 

1314 buffer = io.StringIO() 

1315 self.saveToStream(buffer, skipImports=skipImports) 

1316 return buffer.getvalue() 

1317 

1318 def saveToStream(self, outfile, root="config", skipImports=False): 

1319 """Save a configuration file to a stream, which, when loaded, 

1320 reproduces this config. 

1321 

1322 Parameters 

1323 ---------- 

1324 outfile : file-like object 

1325 Destination file object write the config into. Accepts strings not 

1326 bytes. 

1327 root 

1328 Name to use for the root config variable. The same value must be 

1329 used when loading (see `lsst.pex.config.Config.load`). 

1330 skipImports : `bool`, optional 

1331 If `True` then do not include ``import`` statements in output, 

1332 this is to support human-oriented output from ``pipetask`` where 

1333 additional clutter is not useful. 

1334 

1335 See also 

1336 -------- 

1337 lsst.pex.config.Config.save 

1338 lsst.pex.config.Config.saveToString 

1339 lsst.pex.config.Config.load 

1340 lsst.pex.config.Config.loadFromStream 

1341 lsst.pex.config.Config.loadFromString 

1342 """ 

1343 tmp = self._name 

1344 self._rename(root) 

1345 try: 

1346 if not skipImports: 1346 ↛ 1360line 1346 didn't jump to line 1360, because the condition on line 1346 was never false

1347 self._collectImports() 

1348 # Remove self from the set, as it is handled explicitly below 

1349 self._imports.remove(self.__module__) 

1350 configType = type(self) 

1351 typeString = _typeStr(configType) 

1352 outfile.write(f"import {configType.__module__}\n") 

1353 outfile.write( 

1354 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1355 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1356 ) 

1357 for imp in sorted(self._imports): 1357 ↛ 1358line 1357 didn't jump to line 1358, because the loop on line 1357 never started

1358 if imp in sys.modules and sys.modules[imp] is not None: 

1359 outfile.write("import {}\n".format(imp)) 

1360 self._save(outfile) 

1361 finally: 

1362 self._rename(tmp) 

1363 

1364 def freeze(self): 

1365 """Make this config, and all subconfigs, read-only.""" 

1366 self._frozen = True 

1367 for field in self._fields.values(): 

1368 field.freeze(self) 

1369 

1370 def _save(self, outfile): 

1371 """Save this config to an open stream object. 

1372 

1373 Parameters 

1374 ---------- 

1375 outfile : file-like object 

1376 Destination file object write the config into. Accepts strings not 

1377 bytes. 

1378 """ 

1379 for field in self._fields.values(): 

1380 field.save(outfile, self) 

1381 

1382 def _collectImports(self): 

1383 """Adds module containing self to the list of things to import and 

1384 then loops over all the fields in the config calling a corresponding 

1385 collect method. The field method will call _collectImports on any 

1386 configs it may own and return the set of things to import. This 

1387 returned set will be merged with the set of imports for this config 

1388 class. 

1389 """ 

1390 self._imports.add(self.__module__) 

1391 for name, field in self._fields.items(): 

1392 field._collectImports(self, self._imports) 

1393 

1394 def toDict(self): 

1395 """Make a dictionary of field names and their values. 

1396 

1397 Returns 

1398 ------- 

1399 dict_ : `dict` 

1400 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1401 Values are `~lsst.pex.config.Field` values. 

1402 

1403 See also 

1404 -------- 

1405 lsst.pex.config.Field.toDict 

1406 

1407 Notes 

1408 ----- 

1409 This method uses the `~lsst.pex.config.Field.toDict` method of 

1410 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1411 implement a ``toDict`` method for *this* method to work. 

1412 """ 

1413 dict_ = {} 

1414 for name, field in self._fields.items(): 

1415 dict_[name] = field.toDict(self) 

1416 return dict_ 

1417 

1418 def names(self): 

1419 """Get all the field names in the config, recursively. 

1420 

1421 Returns 

1422 ------- 

1423 names : `list` of `str` 

1424 Field names. 

1425 """ 

1426 # 

1427 # Rather than sort out the recursion all over again use the 

1428 # pre-existing saveToStream() 

1429 # 

1430 with io.StringIO() as strFd: 

1431 self.saveToStream(strFd, "config") 

1432 contents = strFd.getvalue() 

1433 strFd.close() 

1434 # 

1435 # Pull the names out of the dumped config 

1436 # 

1437 keys = [] 

1438 for line in contents.split("\n"): 

1439 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1440 continue 

1441 

1442 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1443 if mat: 

1444 keys.append(mat.group(1)) 

1445 

1446 return keys 

1447 

1448 def _rename(self, name): 

1449 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1450 

1451 Parameters 

1452 ---------- 

1453 name : `str` 

1454 New name for this config in its parent `~lsst.pex.config.Config`. 

1455 

1456 Notes 

1457 ----- 

1458 This method uses the `~lsst.pex.config.Field.rename` method of 

1459 individual `lsst.pex.config.Field` instances. 

1460 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1461 method for *this* method to work. 

1462 

1463 See also 

1464 -------- 

1465 lsst.pex.config.Field.rename 

1466 """ 

1467 self._name = name 

1468 for field in self._fields.values(): 

1469 field.rename(self) 

1470 

1471 def validate(self): 

1472 """Validate the Config, raising an exception if invalid. 

1473 

1474 Raises 

1475 ------ 

1476 lsst.pex.config.FieldValidationError 

1477 Raised if verification fails. 

1478 

1479 Notes 

1480 ----- 

1481 The base class implementation performs type checks on all fields by 

1482 calling their `~lsst.pex.config.Field.validate` methods. 

1483 

1484 Complex single-field validation can be defined by deriving new Field 

1485 types. For convenience, some derived `lsst.pex.config.Field`-types 

1486 (`~lsst.pex.config.ConfigField` and 

1487 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config` 

1488 that handle recursing into subconfigs. 

1489 

1490 Inter-field relationships should only be checked in derived 

1491 `~lsst.pex.config.Config` classes after calling this method, and base 

1492 validation is complete. 

1493 """ 

1494 for field in self._fields.values(): 

1495 field.validate(self) 

1496 

1497 def formatHistory(self, name, **kwargs): 

1498 """Format a configuration field's history to a human-readable format. 

1499 

1500 Parameters 

1501 ---------- 

1502 name : `str` 

1503 Name of a `~lsst.pex.config.Field` in this config. 

1504 kwargs 

1505 Keyword arguments passed to `lsst.pex.config.history.format`. 

1506 

1507 Returns 

1508 ------- 

1509 history : `str` 

1510 A string containing the formatted history. 

1511 

1512 See also 

1513 -------- 

1514 lsst.pex.config.history.format 

1515 """ 

1516 import lsst.pex.config.history as pexHist 

1517 

1518 return pexHist.format(self, name, **kwargs) 

1519 

1520 history = property(lambda x: x._history) 1520 ↛ exitline 1520 didn't run the lambda on line 1520

1521 """Read-only history. 

1522 """ 

1523 

1524 def __setattr__(self, attr, value, at=None, label="assignment"): 

1525 """Set an attribute (such as a field's value). 

1526 

1527 Notes 

1528 ----- 

1529 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1530 locked such that no additional attributes nor properties may be added 

1531 to them dynamically. 

1532 

1533 Although this is not the standard Python behavior, it helps to protect 

1534 users from accidentally mispelling a field name, or trying to set a 

1535 non-existent field. 

1536 """ 

1537 if attr in self._fields: 

1538 if self._fields[attr].deprecated is not None: 1538 ↛ 1539line 1538 didn't jump to line 1539, because the condition on line 1538 was never true

1539 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1540 warnings.warn( 

1541 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1542 FutureWarning, 

1543 stacklevel=2, 

1544 ) 

1545 if at is None: 1545 ↛ 1548line 1545 didn't jump to line 1548, because the condition on line 1545 was never false

1546 at = getCallStack() 

1547 # This allows Field descriptors to work. 

1548 self._fields[attr].__set__(self, value, at=at, label=label) 

1549 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1549 ↛ 1551line 1549 didn't jump to line 1551, because the condition on line 1549 was never true

1550 # This allows properties and other non-Field descriptors to work. 

1551 return object.__setattr__(self, attr, value) 

1552 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1552 ↛ 1557line 1552 didn't jump to line 1557, because the condition on line 1552 was never false

1553 # This allows specific private attributes to work. 

1554 self.__dict__[attr] = value 

1555 else: 

1556 # We throw everything else. 

1557 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr)) 

1558 

1559 def __delattr__(self, attr, at=None, label="deletion"): 

1560 if attr in self._fields: 

1561 if at is None: 

1562 at = getCallStack() 

1563 self._fields[attr].__delete__(self, at=at, label=label) 

1564 else: 

1565 object.__delattr__(self, attr) 

1566 

1567 def __eq__(self, other): 

1568 if type(other) == type(self): 1568 ↛ 1569line 1568 didn't jump to line 1569, because the condition on line 1568 was never true

1569 for name in self._fields: 

1570 thisValue = getattr(self, name) 

1571 otherValue = getattr(other, name) 

1572 if isinstance(thisValue, float) and math.isnan(thisValue): 

1573 if not math.isnan(otherValue): 

1574 return False 

1575 elif thisValue != otherValue: 

1576 return False 

1577 return True 

1578 return False 

1579 

1580 def __ne__(self, other): 

1581 return not self.__eq__(other) 

1582 

1583 def __str__(self): 

1584 return str(self.toDict()) 

1585 

1586 def __repr__(self): 

1587 return "%s(%s)" % ( 

1588 _typeStr(self), 

1589 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None), 

1590 ) 

1591 

1592 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1593 """Compare this configuration to another `~lsst.pex.config.Config` for 

1594 equality. 

1595 

1596 Parameters 

1597 ---------- 

1598 other : `lsst.pex.config.Config` 

1599 Other `~lsst.pex.config.Config` object to compare against this 

1600 config. 

1601 shortcut : `bool`, optional 

1602 If `True`, return as soon as an inequality is found. Default is 

1603 `True`. 

1604 rtol : `float`, optional 

1605 Relative tolerance for floating point comparisons. 

1606 atol : `float`, optional 

1607 Absolute tolerance for floating point comparisons. 

1608 output : callable, optional 

1609 A callable that takes a string, used (possibly repeatedly) to 

1610 report inequalities. 

1611 

1612 Returns 

1613 ------- 

1614 isEqual : `bool` 

1615 `True` when the two `lsst.pex.config.Config` instances are equal. 

1616 `False` if there is an inequality. 

1617 

1618 See also 

1619 -------- 

1620 lsst.pex.config.compareConfigs 

1621 

1622 Notes 

1623 ----- 

1624 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1625 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1626 are not considered by this method. 

1627 

1628 Floating point comparisons are performed by `numpy.allclose`. 

1629 """ 

1630 name1 = self._name if self._name is not None else "config" 

1631 name2 = other._name if other._name is not None else "config" 

1632 name = getComparisonName(name1, name2) 

1633 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1634 

1635 @classmethod 

1636 def __init_subclass__(cls, **kwargs): 

1637 """Run initialization for every subclass. 

1638 

1639 Specifically registers the subclass with a YAML representer 

1640 and YAML constructor (if pyyaml is available) 

1641 """ 

1642 super().__init_subclass__(**kwargs) 

1643 

1644 if not yaml: 1644 ↛ 1645line 1644 didn't jump to line 1645, because the condition on line 1644 was never true

1645 return 

1646 

1647 yaml.add_representer(cls, _yaml_config_representer) 

1648 

1649 @classmethod 

1650 def _fromPython(cls, config_py): 

1651 """Instantiate a `Config`-subclass from serialized Python form. 

1652 

1653 Parameters 

1654 ---------- 

1655 config_py : `str` 

1656 A serialized form of the Config as created by 

1657 `Config.saveToStream`. 

1658 

1659 Returns 

1660 ------- 

1661 config : `Config` 

1662 Reconstructed `Config` instant. 

1663 """ 

1664 cls = _classFromPython(config_py) 

1665 return unreduceConfig(cls, config_py) 

1666 

1667 

1668def _classFromPython(config_py): 

1669 """Return the Config subclass required by this Config serialization. 

1670 

1671 Parameters 

1672 ---------- 

1673 config_py : `str` 

1674 A serialized form of the Config as created by 

1675 `Config.saveToStream`. 

1676 

1677 Returns 

1678 ------- 

1679 cls : `type` 

1680 The `Config` subclass associated with this config. 

1681 """ 

1682 # standard serialization has the form: 

1683 # import config.class 

1684 # assert type(config)==config.class.Config, ... 

1685 # We want to parse these two lines so we can get the class itself 

1686 

1687 # Do a single regex to avoid large string copies when splitting a 

1688 # large config into separate lines. 

1689 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1690 

1691 if not matches: 

1692 first_line, second_line, _ = config_py.split("\n", 2) 

1693 raise ValueError( 

1694 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}" 

1695 ) 

1696 

1697 module_name = matches.group(1) 

1698 module = importlib.import_module(module_name) 

1699 

1700 # Second line 

1701 full_name = matches.group(2) 

1702 

1703 # Remove the module name from the full name 

1704 if not full_name.startswith(module_name): 

1705 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1706 

1707 # if module name is a.b.c and full name is a.b.c.d.E then 

1708 # we need to remove a.b.c. and iterate over the remainder 

1709 # The +1 is for the extra dot after a.b.c 

1710 remainder = full_name[len(module_name) + 1 :] 

1711 components = remainder.split(".") 

1712 pytype = module 

1713 for component in components: 

1714 pytype = getattr(pytype, component) 

1715 return pytype 

1716 

1717 

1718def unreduceConfig(cls, stream): 

1719 """Create a `~lsst.pex.config.Config` from a stream. 

1720 

1721 Parameters 

1722 ---------- 

1723 cls : `lsst.pex.config.Config`-type 

1724 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1725 with configurations in the ``stream``. 

1726 stream : file-like object, `str`, or compiled string 

1727 Stream containing configuration override code. 

1728 

1729 Returns 

1730 ------- 

1731 config : `lsst.pex.config.Config` 

1732 Config instance. 

1733 

1734 See also 

1735 -------- 

1736 lsst.pex.config.Config.loadFromStream 

1737 """ 

1738 config = cls() 

1739 config.loadFromStream(stream) 

1740 return config