Coverage for python/lsst/pex/config/config.py: 58%

458 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-04 21:14 +0000

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from collections.abc import Mapping 

49from typing import Any, ForwardRef, Generic, TypeVar, cast, overload 

50 

51try: 

52 from types import GenericAlias 

53except ImportError: 

54 # cover python 3.8 usage 

55 GenericAlias = type(Mapping[int, int]) 

56 

57# if YAML is not available that's fine and we simply don't register 

58# the yaml representer since we know it won't be used. 

59try: 

60 import yaml 

61except ImportError: 

62 yaml = None 

63 

64from .callStack import getCallStack, getStackFrame 

65from .comparison import compareConfigs, compareScalars, getComparisonName 

66 

67if yaml: 67 ↛ 78line 67 didn't jump to line 78, because the condition on line 67 was never false

68 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

69 

70 try: 

71 # CLoader is not always available 

72 from yaml import CLoader 

73 

74 YamlLoaders += (CLoader,) 

75 except ImportError: 

76 pass 

77else: 

78 YamlLoaders = () 

79 doImport = None 

80 

81 

82class _PexConfigGenericAlias(GenericAlias): 

83 """A Subclass of python's GenericAlias used in defining and instantiating 

84 Generics. 

85 

86 This class differs from `types.GenericAlias` in that it calls a method 

87 named _parseTypingArgs defined on Fields. This method gives Field and its 

88 subclasses an opportunity to transform type parameters into class key word 

89 arguments. Code authors do not need to implement any returns of this object 

90 directly, and instead only need implement _parseTypingArgs, if a Field 

91 subclass differs from the base class implementation. 

92 

93 This class is intended to be an implementation detail, returned from a 

94 Field's `__class_getitem__` method. 

95 """ 

96 

97 def __call__(self, *args: Any, **kwds: Any) -> Any: 

98 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

99 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

100 

101 

102FieldTypeVar = TypeVar("FieldTypeVar") 

103 

104 

105class UnexpectedProxyUsageError(TypeError): 

106 """Exception raised when a proxy class is used in a context that suggests 

107 it should have already been converted to the thing it proxies. 

108 """ 

109 

110 

111def _joinNamePath(prefix=None, name=None, index=None): 

112 """Generate nested configuration names.""" 

113 if not prefix and not name: 113 ↛ 114line 113 didn't jump to line 114, because the condition on line 113 was never true

114 raise ValueError("Invalid name: cannot be None") 

115 elif not name: 115 ↛ 116line 115 didn't jump to line 116, because the condition on line 115 was never true

116 name = prefix 

117 elif prefix and name: 117 ↛ 120line 117 didn't jump to line 120, because the condition on line 117 was never false

118 name = prefix + "." + name 

119 

120 if index is not None: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 return f"{name}[{index!r}]" 

122 else: 

123 return name 

124 

125 

126def _autocast(x, dtype): 

127 """Cast a value to a type, if appropriate. 

128 

129 Parameters 

130 ---------- 

131 x : object 

132 A value. 

133 dtype : tpye 

134 Data type, such as `float`, `int`, or `str`. 

135 

136 Returns 

137 ------- 

138 values : object 

139 If appropriate, the returned value is ``x`` cast to the given type 

140 ``dtype``. If the cast cannot be performed the original value of 

141 ``x`` is returned. 

142 """ 

143 if dtype == float and isinstance(x, int): 

144 return float(x) 

145 return x 

146 

147 

148def _typeStr(x): 

149 """Generate a fully-qualified type name. 

150 

151 Returns 

152 ------- 

153 `str` 

154 Fully-qualified type name. 

155 

156 Notes 

157 ----- 

158 This function is used primarily for writing config files to be executed 

159 later upon with the 'load' function. 

160 """ 

161 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

162 xtype = x 

163 else: 

164 xtype = type(x) 

165 if xtype.__module__ == "builtins": 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true

166 return xtype.__name__ 

167 else: 

168 return f"{xtype.__module__}.{xtype.__name__}" 

169 

170 

171if yaml: 171 ↛ 204line 171 didn't jump to line 204, because the condition on line 171 was never false

172 

173 def _yaml_config_representer(dumper, data): 

174 """Represent a Config object in a form suitable for YAML. 

175 

176 Stores the serialized stream as a scalar block string. 

177 """ 

178 stream = io.StringIO() 

179 data.saveToStream(stream) 

180 config_py = stream.getvalue() 

181 

182 # Strip multiple newlines from the end of the config 

183 # This simplifies the YAML to use | and not |+ 

184 config_py = config_py.rstrip() + "\n" 

185 

186 # Trailing spaces force pyyaml to use non-block form. 

187 # Remove the trailing spaces so it has no choice 

188 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

189 

190 # Store the Python as a simple scalar 

191 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

192 

193 def _yaml_config_constructor(loader, node): 

194 """Construct a config from YAML.""" 

195 config_py = loader.construct_scalar(node) 

196 return Config._fromPython(config_py) 

197 

198 # Register a generic constructor for Config and all subclasses 

199 # Need to register for all the loaders we would like to use 

200 for loader in YamlLoaders: 

201 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

202 

203 

204class ConfigMeta(type): 

205 """A metaclass for `lsst.pex.config.Config`. 

206 

207 Notes 

208 ----- 

209 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

210 class attributes as a class attribute called ``_fields``, and adds 

211 the name of each field as an instance variable of the field itself (so you 

212 don't have to pass the name of the field to the field constructor). 

213 """ 

214 

215 def __init__(cls, name, bases, dict_): 

216 type.__init__(cls, name, bases, dict_) 

217 cls._fields = {} 

218 cls._source = getStackFrame() 

219 

220 def getFields(classtype): 

221 fields = {} 

222 bases = list(classtype.__bases__) 

223 bases.reverse() 

224 for b in bases: 

225 fields.update(getFields(b)) 

226 

227 for k, v in classtype.__dict__.items(): 

228 if isinstance(v, Field): 

229 fields[k] = v 

230 return fields 

231 

232 fields = getFields(cls) 

233 for k, v in fields.items(): 

234 setattr(cls, k, copy.deepcopy(v)) 

235 

236 def __setattr__(cls, name, value): 

237 if isinstance(value, Field): 

238 value.name = name 

239 cls._fields[name] = value 

240 type.__setattr__(cls, name, value) 

241 

242 

243class FieldValidationError(ValueError): 

244 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

245 particular ``~lsst.pex.config.Config``. 

246 

247 Parameters 

248 ---------- 

249 field : `lsst.pex.config.Field` 

250 The field that was not valid. 

251 config : `lsst.pex.config.Config` 

252 The config containing the invalid field. 

253 msg : `str` 

254 Text describing why the field was not valid. 

255 """ 

256 

257 def __init__(self, field, config, msg): 

258 self.fieldType = type(field) 

259 """Type of the `~lsst.pex.config.Field` that incurred the error. 

260 """ 

261 

262 self.fieldName = field.name 

263 """Name of the `~lsst.pex.config.Field` instance that incurred the 

264 error (`str`). 

265 

266 See also 

267 -------- 

268 lsst.pex.config.Field.name 

269 """ 

270 

271 self.fullname = _joinNamePath(config._name, field.name) 

272 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

273 (`str`). 

274 """ 

275 

276 self.history = config.history.setdefault(field.name, []) 

277 """Full history of all changes to the `~lsst.pex.config.Field` 

278 instance. 

279 """ 

280 

281 self.fieldSource = field.source 

282 """File and line number of the `~lsst.pex.config.Field` definition. 

283 """ 

284 

285 self.configSource = config._source 

286 error = ( 

287 "%s '%s' failed validation: %s\n" 

288 "For more information see the Field definition at:\n%s" 

289 " and the Config definition at:\n%s" 

290 % ( 

291 self.fieldType.__name__, 

292 self.fullname, 

293 msg, 

294 self.fieldSource.format(), 

295 self.configSource.format(), 

296 ) 

297 ) 

298 super().__init__(error) 

299 

300 

301class Field(Generic[FieldTypeVar]): 

302 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

303 `complex`, `bool`, and `str` data types. 

304 

305 Parameters 

306 ---------- 

307 doc : `str` 

308 A description of the field for users. 

309 dtype : type, optional 

310 The field's data type. ``Field`` only supports basic data types: 

311 `int`, `float`, `complex`, `bool`, and `str`. See 

312 `Field.supportedTypes`. Optional if supplied as a typing argument to 

313 the class. 

314 default : object, optional 

315 The field's default value. 

316 check : callable, optional 

317 A callable that is called with the field's value. This callable should 

318 return `False` if the value is invalid. More complex inter-field 

319 validation can be written as part of the 

320 `lsst.pex.config.Config.validate` method. 

321 optional : `bool`, optional 

322 This sets whether the field is considered optional, and therefore 

323 doesn't need to be set by the user. When `False`, 

324 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

325 deprecated : None or `str`, optional 

326 A description of why this Field is deprecated, including removal date. 

327 If not None, the string is appended to the docstring for this Field. 

328 

329 Raises 

330 ------ 

331 ValueError 

332 Raised when the ``dtype`` parameter is not one of the supported types 

333 (see `Field.supportedTypes`). 

334 

335 See Also 

336 -------- 

337 ChoiceField 

338 ConfigChoiceField 

339 ConfigDictField 

340 ConfigField 

341 ConfigurableField 

342 DictField 

343 ListField 

344 RangeField 

345 RegistryField 

346 

347 Notes 

348 ----- 

349 ``Field`` instances (including those of any subclass of ``Field``) are used 

350 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

351 example, below). ``Field`` attributes work like the `property` attributes 

352 of classes that implement custom setters and getters. `Field` attributes 

353 belong to the class, but operate on the instance. Formally speaking, 

354 `Field` attributes are `descriptors 

355 <https://docs.python.org/3/howto/descriptor.html>`_. 

356 

357 When you access a `Field` attribute on a `Config` instance, you don't 

358 get the `Field` instance itself. Instead, you get the value of that field, 

359 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

360 container type (like a `lsst.pex.config.List`) depending on the field's 

361 type. See the example, below. 

362 

363 Fields can be annotated with a type similar to other python classes (python 

364 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

365 See the name field in the Config example below for an example of this. 

366 Unlike most other uses in python, this has an effect at type checking *and* 

367 runtime. If the type is specified with a class annotation, it will be used 

368 as the value of the ``dtype`` in the ``Field`` and there is no need to 

369 specify it as an argument during instantiation. 

370 

371 There are Some notes on dtype through type annotation syntax. Type 

372 annotation syntax supports supplying the argument as a string of a type 

373 name. i.e. "float", but this cannot be used to resolve circular references. 

374 Type annotation syntax can be used on an identifier in addition to Class 

375 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

376 ``someField = Field[str](doc="some doc"). However, this syntax is only 

377 useful for annotating the type of the identifier (i.e. variable in previous 

378 example) and does nothing for assigning the dtype of the ``Field``. 

379 

380 

381 Examples 

382 -------- 

383 Instances of ``Field`` should be used as class attributes of 

384 `lsst.pex.config.Config` subclasses: 

385 

386 >>> from lsst.pex.config import Config, Field 

387 >>> class Example(Config): 

388 ... myInt = Field("An integer field.", int, default=0) 

389 ... name = Field[str](doc="A string Field") 

390 ... 

391 >>> print(config.myInt) 

392 0 

393 >>> config.myInt = 5 

394 >>> print(config.myInt) 

395 5 

396 """ 

397 

398 name: str 

399 """Identifier (variable name) used to refer to a Field within a Config 

400 Class. 

401 """ 

402 

403 supportedTypes = {str, bool, float, int, complex} 

404 """Supported data types for field values (`set` of types). 

405 """ 

406 

407 @staticmethod 

408 def _parseTypingArgs( 

409 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any] 

410 ) -> Mapping[str, Any]: 

411 """Parse type annotations into keyword constructor arguments. 

412 

413 This is a special private method that interprets type arguments (i.e. 

414 Field[str]) into keyword arguments to be passed on to the constructor. 

415 

416 Subclasses of Field can implement this method to customize how they 

417 handle turning type parameters into keyword arguments (see DictField 

418 for an example) 

419 

420 Parameters 

421 ---------- 

422 params : `tuple` of `type` or `tuple` of str 

423 Parameters passed to the type annotation. These will either be 

424 types or strings. Strings are to interpreted as forward references 

425 and will be treated as such. 

426 kwds : `MutableMapping` with keys of `str` and values of `Any` 

427 These are the user supplied keywords that are to be passed to the 

428 Field constructor. 

429 

430 Returns 

431 ------- 

432 kwds : `MutableMapping` with keys of `str` and values of `Any` 

433 The mapping of keywords that will be passed onto the constructor 

434 of the Field. Should be filled in with any information gleaned 

435 from the input parameters. 

436 

437 Raises 

438 ------ 

439 ValueError : 

440 Raised if params is of incorrect length. 

441 Raised if a forward reference could not be resolved 

442 Raised if there is a conflict between params and values in kwds 

443 """ 

444 if len(params) > 1: 

445 raise ValueError("Only single type parameters are supported") 

446 unpackedParams = params[0] 

447 if isinstance(unpackedParams, str): 

448 _typ = ForwardRef(unpackedParams) 

449 # type ignore below because typeshed seems to be wrong. It 

450 # indicates there are only 2 args, as it was in python 3.8, but 

451 # 3.9+ takes 3 args. Attempt in old style and new style to 

452 # work with both. 

453 try: 

454 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

455 except TypeError: 

456 # python 3.8 path 

457 result = _typ._evaluate(globals(), locals()) 

458 if result is None: 

459 raise ValueError("Could not deduce type from input") 

460 unpackedParams = cast(type, result) 

461 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

462 raise ValueError("Conflicting definition for dtype") 

463 elif "dtype" not in kwds: 

464 kwds = {**kwds, **{"dtype": unpackedParams}} 

465 return kwds 

466 

467 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef): 

468 return _PexConfigGenericAlias(cls, params) 

469 

470 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

471 if dtype is None: 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 raise ValueError( 

473 "dtype must either be supplied as an argument or as a type argument to the class" 

474 ) 

475 if dtype not in self.supportedTypes: 475 ↛ 476line 475 didn't jump to line 476, because the condition on line 475 was never true

476 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

477 

478 source = getStackFrame() 

479 self._setup( 

480 doc=doc, 

481 dtype=dtype, 

482 default=default, 

483 check=check, 

484 optional=optional, 

485 source=source, 

486 deprecated=deprecated, 

487 ) 

488 

489 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

490 """Set attributes, usually during initialization.""" 

491 self.dtype = dtype 

492 """Data type for the field. 

493 """ 

494 

495 if not doc: 495 ↛ 496line 495 didn't jump to line 496, because the condition on line 495 was never true

496 raise ValueError("Docstring is empty.") 

497 

498 # append the deprecation message to the docstring. 

499 if deprecated is not None: 

500 doc = f"{doc} Deprecated: {deprecated}" 

501 self.doc = doc 

502 """A description of the field (`str`). 

503 """ 

504 

505 self.deprecated = deprecated 

506 """If not None, a description of why this field is deprecated (`str`). 

507 """ 

508 

509 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

510 if optional or default is not None: 

511 self.__doc__ += f", default ``{default!r}``" 

512 self.__doc__ += ")" 

513 

514 self.default = default 

515 """Default value for this field. 

516 """ 

517 

518 self.check = check 

519 """A user-defined function that validates the value of the field. 

520 """ 

521 

522 self.optional = optional 

523 """Flag that determines if the field is required to be set (`bool`). 

524 

525 When `False`, `lsst.pex.config.Config.validate` will fail if the 

526 field's value is `None`. 

527 """ 

528 

529 self.source = source 

530 """The stack frame where this field is defined (`list` of 

531 `lsst.pex.config.callStack.StackFrame`). 

532 """ 

533 

534 def rename(self, instance): 

535 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use 

536 only). 

537 

538 Parameters 

539 ---------- 

540 instance : `lsst.pex.config.Config` 

541 The config instance that contains this field. 

542 

543 Notes 

544 ----- 

545 This method is invoked by the `lsst.pex.config.Config` object that 

546 contains this field and should not be called directly. 

547 

548 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

549 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should 

550 rename each subconfig with the full field name as generated by 

551 `lsst.pex.config.config._joinNamePath`. 

552 """ 

553 pass 

554 

555 def validate(self, instance): 

556 """Validate the field (for internal use only). 

557 

558 Parameters 

559 ---------- 

560 instance : `lsst.pex.config.Config` 

561 The config instance that contains this field. 

562 

563 Raises 

564 ------ 

565 lsst.pex.config.FieldValidationError 

566 Raised if verification fails. 

567 

568 Notes 

569 ----- 

570 This method provides basic validation: 

571 

572 - Ensures that the value is not `None` if the field is not optional. 

573 - Ensures type correctness. 

574 - Ensures that the user-provided ``check`` function is valid. 

575 

576 Most `~lsst.pex.config.Field` subclasses should call 

577 `lsst.pex.config.Field.validate` if they re-implement 

578 `~lsst.pex.config.Field.validate`. 

579 """ 

580 value = self.__get__(instance) 

581 if not self.optional and value is None: 

582 raise FieldValidationError(self, instance, "Required value cannot be None") 

583 

584 def freeze(self, instance): 

585 """Make this field read-only (for internal use only). 

586 

587 Parameters 

588 ---------- 

589 instance : `lsst.pex.config.Config` 

590 The config instance that contains this field. 

591 

592 Notes 

593 ----- 

594 Freezing is only relevant for fields that hold subconfigs. Fields which 

595 hold subconfigs should freeze each subconfig. 

596 

597 **Subclasses should implement this method.** 

598 """ 

599 pass 

600 

601 def _validateValue(self, value): 

602 """Validate a value. 

603 

604 Parameters 

605 ---------- 

606 value : object 

607 The value being validated. 

608 

609 Raises 

610 ------ 

611 TypeError 

612 Raised if the value's type is incompatible with the field's 

613 ``dtype``. 

614 ValueError 

615 Raised if the value is rejected by the ``check`` method. 

616 """ 

617 if value is None: 617 ↛ 618line 617 didn't jump to line 618, because the condition on line 617 was never true

618 return 

619 

620 if not isinstance(value, self.dtype): 620 ↛ 621line 620 didn't jump to line 621, because the condition on line 620 was never true

621 msg = "Value {} is of incorrect type {}. Expected type {}".format( 

622 value, 

623 _typeStr(value), 

624 _typeStr(self.dtype), 

625 ) 

626 raise TypeError(msg) 

627 if self.check is not None and not self.check(value): 627 ↛ 628line 627 didn't jump to line 628, because the condition on line 627 was never true

628 msg = "Value %s is not a valid value" % str(value) 

629 raise ValueError(msg) 

630 

631 def _collectImports(self, instance, imports): 

632 """Call the _collectImports method on all config 

633 objects the field may own, and union them with the supplied imports 

634 set. 

635 

636 Parameters 

637 ---------- 

638 instance : instance or subclass of `lsst.pex.config.Config` 

639 A config object that has this field defined on it 

640 imports : `set` 

641 Set of python modules that need imported after persistence 

642 """ 

643 pass 

644 

645 def save(self, outfile, instance): 

646 """Save this field to a file (for internal use only). 

647 

648 Parameters 

649 ---------- 

650 outfile : file-like object 

651 A writeable field handle. 

652 instance : `~lsst.pex.config.Config` 

653 The `~lsst.pex.config.Config` instance that contains this field. 

654 

655 Notes 

656 ----- 

657 This method is invoked by the `~lsst.pex.config.Config` object that 

658 contains this field and should not be called directly. 

659 

660 The output consists of the documentation string 

661 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

662 line is formatted as an assignment: ``{fullname}={value}``. 

663 

664 This output can be executed with Python. 

665 """ 

666 value = self.__get__(instance) 

667 fullname = _joinNamePath(instance._name, self.name) 

668 

669 if self.deprecated and value == self.default: 669 ↛ 670line 669 didn't jump to line 670, because the condition on line 669 was never true

670 return 

671 

672 # write full documentation string as comment lines 

673 # (i.e. first character is #) 

674 doc = "# " + str(self.doc).replace("\n", "\n# ") 

675 if isinstance(value, float) and not math.isfinite(value): 675 ↛ 677line 675 didn't jump to line 677, because the condition on line 675 was never true

676 # non-finite numbers need special care 

677 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n") 

678 else: 

679 outfile.write(f"{doc}\n{fullname}={value!r}\n\n") 

680 

681 def toDict(self, instance): 

682 """Convert the field value so that it can be set as the value of an 

683 item in a `dict` (for internal use only). 

684 

685 Parameters 

686 ---------- 

687 instance : `~lsst.pex.config.Config` 

688 The `~lsst.pex.config.Config` that contains this field. 

689 

690 Returns 

691 ------- 

692 value : object 

693 The field's value. See *Notes*. 

694 

695 Notes 

696 ----- 

697 This method invoked by the owning `~lsst.pex.config.Config` object and 

698 should not be called directly. 

699 

700 Simple values are passed through. Complex data structures must be 

701 manipulated. For example, a `~lsst.pex.config.Field` holding a 

702 subconfig should, instead of the subconfig object, return a `dict` 

703 where the keys are the field names in the subconfig, and the values are 

704 the field values in the subconfig. 

705 """ 

706 return self.__get__(instance) 

707 

708 @overload 

709 def __get__( 

710 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

711 ) -> Field[FieldTypeVar]: 

712 ... 

713 

714 @overload 

715 def __get__( 

716 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default" 

717 ) -> FieldTypeVar: 

718 ... 

719 

720 def __get__(self, instance, owner=None, at=None, label="default"): 

721 """Define how attribute access should occur on the Config instance 

722 This is invoked by the owning config object and should not be called 

723 directly. 

724 

725 When the field attribute is accessed on a Config class object, it 

726 returns the field object itself in order to allow inspection of 

727 Config classes. 

728 

729 When the field attribute is access on a config instance, the actual 

730 value described by the field (and held by the Config instance) is 

731 returned. 

732 """ 

733 if instance is None: 733 ↛ 734line 733 didn't jump to line 734, because the condition on line 733 was never true

734 return self 

735 else: 

736 # try statements are almost free in python if they succeed 

737 try: 

738 return instance._storage[self.name] 

739 except AttributeError: 

740 if not isinstance(instance, Config): 

741 return self 

742 else: 

743 raise AttributeError( 

744 f"Config {instance} is missing _storage attribute, likely incorrectly initialized" 

745 ) 

746 

747 def __set__( 

748 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment" 

749 ) -> None: 

750 """Set an attribute on the config instance. 

751 

752 Parameters 

753 ---------- 

754 instance : `lsst.pex.config.Config` 

755 The config instance that contains this field. 

756 value : obj 

757 Value to set on this field. 

758 at : `list` of `lsst.pex.config.callStack.StackFrame` 

759 The call stack (created by 

760 `lsst.pex.config.callStack.getCallStack`). 

761 label : `str`, optional 

762 Event label for the history. 

763 

764 Notes 

765 ----- 

766 This method is invoked by the owning `lsst.pex.config.Config` object 

767 and should not be called directly. 

768 

769 Derived `~lsst.pex.config.Field` classes may need to override the 

770 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

771 should follow the following rules: 

772 

773 - Do not allow modification of frozen configs. 

774 - Validate the new value **before** modifying the field. Except if the 

775 new value is `None`. `None` is special and no attempt should be made 

776 to validate it until `lsst.pex.config.Config.validate` is called. 

777 - Do not modify the `~lsst.pex.config.Config` instance to contain 

778 invalid values. 

779 - If the field is modified, update the history of the 

780 `lsst.pex.config.field.Field` to reflect the changes. 

781 

782 In order to decrease the need to implement this method in derived 

783 `~lsst.pex.config.Field` types, value validation is performed in the 

784 `lsst.pex.config.Field._validateValue`. If only the validation step 

785 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

786 implement `lsst.pex.config.Field._validateValue` than to reimplement 

787 ``__set__``. More complicated behavior, however, may require 

788 reimplementation. 

789 """ 

790 if instance._frozen: 790 ↛ 791line 790 didn't jump to line 791, because the condition on line 790 was never true

791 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

792 

793 history = instance._history.setdefault(self.name, []) 

794 if value is not None: 794 ↛ 801line 794 didn't jump to line 801, because the condition on line 794 was never false

795 value = _autocast(value, self.dtype) 

796 try: 

797 self._validateValue(value) 

798 except BaseException as e: 

799 raise FieldValidationError(self, instance, str(e)) 

800 

801 instance._storage[self.name] = value 

802 if at is None: 802 ↛ 803line 802 didn't jump to line 803, because the condition on line 802 was never true

803 at = getCallStack() 

804 history.append((value, at, label)) 

805 

806 def __delete__(self, instance, at=None, label="deletion"): 

807 """Delete an attribute from a `lsst.pex.config.Config` instance. 

808 

809 Parameters 

810 ---------- 

811 instance : `lsst.pex.config.Config` 

812 The config instance that contains this field. 

813 at : `list` of `lsst.pex.config.callStack.StackFrame` 

814 The call stack (created by 

815 `lsst.pex.config.callStack.getCallStack`). 

816 label : `str`, optional 

817 Event label for the history. 

818 

819 Notes 

820 ----- 

821 This is invoked by the owning `~lsst.pex.config.Config` object and 

822 should not be called directly. 

823 """ 

824 if at is None: 

825 at = getCallStack() 

826 self.__set__(instance, None, at=at, label=label) 

827 

828 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

829 """Compare a field (named `Field.name`) in two 

830 `~lsst.pex.config.Config` instances for equality. 

831 

832 Parameters 

833 ---------- 

834 instance1 : `lsst.pex.config.Config` 

835 Left-hand side `Config` instance to compare. 

836 instance2 : `lsst.pex.config.Config` 

837 Right-hand side `Config` instance to compare. 

838 shortcut : `bool`, optional 

839 **Unused.** 

840 rtol : `float`, optional 

841 Relative tolerance for floating point comparisons. 

842 atol : `float`, optional 

843 Absolute tolerance for floating point comparisons. 

844 output : callable, optional 

845 A callable that takes a string, used (possibly repeatedly) to 

846 report inequalities. 

847 

848 Notes 

849 ----- 

850 This method must be overridden by more complex `Field` subclasses. 

851 

852 See Also 

853 -------- 

854 lsst.pex.config.compareScalars 

855 """ 

856 v1 = getattr(instance1, self.name) 

857 v2 = getattr(instance2, self.name) 

858 name = getComparisonName( 

859 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

860 ) 

861 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

862 

863 

864class RecordingImporter: 

865 """Importer (for `sys.meta_path`) that records which modules are being 

866 imported. 

867 

868 *This class does not do any importing itself.* 

869 

870 Examples 

871 -------- 

872 Use this class as a context manager to ensure it is properly uninstalled 

873 when done: 

874 

875 >>> with RecordingImporter() as importer: 

876 ... # import stuff 

877 ... import numpy as np 

878 ... print("Imported: " + importer.getModules()) 

879 """ 

880 

881 def __init__(self): 

882 self._modules = set() 

883 

884 def __enter__(self): 

885 self.origMetaPath = sys.meta_path 

886 sys.meta_path = [self] + sys.meta_path # type: ignore 

887 return self 

888 

889 def __exit__(self, *args): 

890 self.uninstall() 

891 return False # Don't suppress exceptions 

892 

893 def uninstall(self): 

894 """Uninstall the importer.""" 

895 sys.meta_path = self.origMetaPath 

896 

897 def find_spec(self, fullname, path, target=None): 

898 """Find a module. 

899 

900 Called as part of the ``import`` chain of events. 

901 """ 

902 self._modules.add(fullname) 

903 # Return None because we don't do any importing. 

904 return None 

905 

906 def getModules(self): 

907 """Get the set of modules that were imported. 

908 

909 Returns 

910 ------- 

911 modules : `set` of `str` 

912 Set of imported module names. 

913 """ 

914 return self._modules 

915 

916 

917# type ignore because type checker thinks ConfigMeta is Generic when it is not 

918class Config(metaclass=ConfigMeta): # type: ignore 

919 """Base class for configuration (*config*) objects. 

920 

921 Notes 

922 ----- 

923 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

924 instances as class attributes. These are used to define most of the base 

925 class behavior. 

926 

927 ``Config`` implements a mapping API that provides many `dict`-like methods, 

928 such as `keys`, `values`, and `items`. ``Config`` instances also support 

929 the ``in`` operator to test if a field is in the config. Unlike a `dict`, 

930 ``Config`` classes are not subscriptable. Instead, access individual 

931 fields as attributes of the configuration instance. 

932 

933 Examples 

934 -------- 

935 Config classes are subclasses of ``Config`` that have 

936 `~lsst.pex.config.Field` instances (or instances of 

937 `~lsst.pex.config.Field` subclasses) as class attributes: 

938 

939 >>> from lsst.pex.config import Config, Field, ListField 

940 >>> class DemoConfig(Config): 

941 ... intField = Field(doc="An integer field", dtype=int, default=42) 

942 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

943 ... default=['coffee', 'green tea', 'water']) 

944 ... 

945 >>> config = DemoConfig() 

946 

947 Configs support many `dict`-like APIs: 

948 

949 >>> config.keys() 

950 ['intField', 'listField'] 

951 >>> 'intField' in config 

952 True 

953 

954 Individual fields can be accessed as attributes of the configuration: 

955 

956 >>> config.intField 

957 42 

958 >>> config.listField.append('earl grey tea') 

959 >>> print(config.listField) 

960 ['coffee', 'green tea', 'water', 'earl grey tea'] 

961 """ 

962 

963 _storage: dict[str, Any] 

964 _fields: dict[str, Field] 

965 _history: dict[str, list[Any]] 

966 _imports: set[Any] 

967 

968 def __iter__(self): 

969 """Iterate over fields.""" 

970 return self._fields.__iter__() 

971 

972 def keys(self): 

973 """Get field names. 

974 

975 Returns 

976 ------- 

977 names : `~collections.abc.KeysView` 

978 List of `lsst.pex.config.Field` names. 

979 """ 

980 return self._storage.keys() 

981 

982 def values(self): 

983 """Get field values. 

984 

985 Returns 

986 ------- 

987 values : `~collections.abc.ValuesView` 

988 Iterator of field values. 

989 """ 

990 return self._storage.values() 

991 

992 def items(self): 

993 """Get configurations as ``(field name, field value)`` pairs. 

994 

995 Returns 

996 ------- 

997 items : `~collections.abc.ItemsView` 

998 Iterator of tuples for each configuration. Tuple items are: 

999 

1000 0. Field name. 

1001 1. Field value. 

1002 """ 

1003 return self._storage.items() 

1004 

1005 def __contains__(self, name): 

1006 """Return `True` if the specified field exists in this config. 

1007 

1008 Parameters 

1009 ---------- 

1010 name : `str` 

1011 Field name to test for. 

1012 

1013 Returns 

1014 ------- 

1015 in : `bool` 

1016 `True` if the specified field exists in the config. 

1017 """ 

1018 return self._storage.__contains__(name) 

1019 

1020 def __new__(cls, *args, **kw): 

1021 """Allocate a new `lsst.pex.config.Config` object. 

1022 

1023 In order to ensure that all Config object are always in a proper state 

1024 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1025 some attributes are handled at allocation time rather than at 

1026 initialization. 

1027 

1028 This ensures that even if a derived `~lsst.pex.config.Config` class 

1029 implements ``__init__``, its author does not need to be concerned about 

1030 when or even the base ``Config.__init__`` should be called. 

1031 """ 

1032 name = kw.pop("__name", None) 

1033 at = kw.pop("__at", getCallStack()) 

1034 # remove __label and ignore it 

1035 kw.pop("__label", "default") 

1036 

1037 instance = object.__new__(cls) 

1038 instance._frozen = False 

1039 instance._name = name 

1040 instance._storage = {} 

1041 instance._history = {} 

1042 instance._imports = set() 

1043 # load up defaults 

1044 for field in instance._fields.values(): 

1045 instance._history[field.name] = [] 

1046 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1047 # set custom default-overrides 

1048 instance.setDefaults() 

1049 # set constructor overrides 

1050 instance.update(__at=at, **kw) 

1051 return instance 

1052 

1053 def __reduce__(self): 

1054 """Reduction for pickling (function with arguments to reproduce). 

1055 

1056 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1057 since it may contain lambdas (as the ``check`` elements) that cannot 

1058 be pickled. 

1059 """ 

1060 # The stream must be in characters to match the API but pickle 

1061 # requires bytes 

1062 stream = io.StringIO() 

1063 self.saveToStream(stream) 

1064 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1065 

1066 def setDefaults(self): 

1067 """Subclass hook for computing defaults. 

1068 

1069 Notes 

1070 ----- 

1071 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1072 rather than using the `~lsst.pex.config.Field` instances's defaults 

1073 should do so here. To correctly use inherited defaults, 

1074 implementations of ``setDefaults`` must call their base class's 

1075 ``setDefaults``. 

1076 """ 

1077 pass 

1078 

1079 def update(self, **kw): 

1080 """Update values of fields specified by the keyword arguments. 

1081 

1082 Parameters 

1083 ---------- 

1084 kw 

1085 Keywords are configuration field names. Values are configuration 

1086 field values. 

1087 

1088 Notes 

1089 ----- 

1090 The ``__at`` and ``__label`` keyword arguments are special internal 

1091 keywords. They are used to strip out any internal steps from the 

1092 history tracebacks of the config. Do not modify these keywords to 

1093 subvert a `~lsst.pex.config.Config` instance's history. 

1094 

1095 Examples 

1096 -------- 

1097 This is a config with three fields: 

1098 

1099 >>> from lsst.pex.config import Config, Field 

1100 >>> class DemoConfig(Config): 

1101 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1102 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1103 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1104 ... 

1105 >>> config = DemoConfig() 

1106 

1107 These are the default values of each field: 

1108 

1109 >>> for name, value in config.iteritems(): 

1110 ... print(f"{name}: {value}") 

1111 ... 

1112 fieldA: 42 

1113 fieldB: True 

1114 fieldC: 'Hello world' 

1115 

1116 Using this method to update ``fieldA`` and ``fieldC``: 

1117 

1118 >>> config.update(fieldA=13, fieldC='Updated!') 

1119 

1120 Now the values of each field are: 

1121 

1122 >>> for name, value in config.iteritems(): 

1123 ... print(f"{name}: {value}") 

1124 ... 

1125 fieldA: 13 

1126 fieldB: True 

1127 fieldC: 'Updated!' 

1128 """ 

1129 at = kw.pop("__at", getCallStack()) 

1130 label = kw.pop("__label", "update") 

1131 

1132 for name, value in kw.items(): 

1133 try: 

1134 field = self._fields[name] 

1135 field.__set__(self, value, at=at, label=label) 

1136 except KeyError: 

1137 raise KeyError(f"No field of name {name} exists in config type {_typeStr(self)}") 

1138 

1139 def load(self, filename, root="config"): 

1140 """Modify this config in place by executing the Python code in a 

1141 configuration file. 

1142 

1143 Parameters 

1144 ---------- 

1145 filename : `str` 

1146 Name of the configuration file. A configuration file is Python 

1147 module. 

1148 root : `str`, optional 

1149 Name of the variable in file that refers to the config being 

1150 overridden. 

1151 

1152 For example, the value of root is ``"config"`` and the file 

1153 contains:: 

1154 

1155 config.myField = 5 

1156 

1157 Then this config's field ``myField`` is set to ``5``. 

1158 

1159 See Also 

1160 -------- 

1161 lsst.pex.config.Config.loadFromStream 

1162 lsst.pex.config.Config.loadFromString 

1163 lsst.pex.config.Config.save 

1164 lsst.pex.config.Config.saveToStream 

1165 lsst.pex.config.Config.saveToString 

1166 """ 

1167 with open(filename) as f: 

1168 code = compile(f.read(), filename=filename, mode="exec") 

1169 self.loadFromString(code, root=root, filename=filename) 

1170 

1171 def loadFromStream(self, stream, root="config", filename=None): 

1172 """Modify this Config in place by executing the Python code in the 

1173 provided stream. 

1174 

1175 Parameters 

1176 ---------- 

1177 stream : file-like object, `str`, `bytes`, or compiled string 

1178 Stream containing configuration override code. If this is a 

1179 code object, it should be compiled with ``mode="exec"``. 

1180 root : `str`, optional 

1181 Name of the variable in file that refers to the config being 

1182 overridden. 

1183 

1184 For example, the value of root is ``"config"`` and the file 

1185 contains:: 

1186 

1187 config.myField = 5 

1188 

1189 Then this config's field ``myField`` is set to ``5``. 

1190 filename : `str`, optional 

1191 Name of the configuration file, or `None` if unknown or contained 

1192 in the stream. Used for error reporting. 

1193 

1194 Notes 

1195 ----- 

1196 For backwards compatibility reasons, this method accepts strings, bytes 

1197 and code objects as well as file-like objects. New code should use 

1198 `loadFromString` instead for most of these types. 

1199 

1200 See Also 

1201 -------- 

1202 lsst.pex.config.Config.load 

1203 lsst.pex.config.Config.loadFromString 

1204 lsst.pex.config.Config.save 

1205 lsst.pex.config.Config.saveToStream 

1206 lsst.pex.config.Config.saveToString 

1207 """ 

1208 if hasattr(stream, "read"): 1208 ↛ 1209line 1208 didn't jump to line 1209, because the condition on line 1208 was never true

1209 if filename is None: 

1210 filename = getattr(stream, "name", "?") 

1211 code = compile(stream.read(), filename=filename, mode="exec") 

1212 else: 

1213 code = stream 

1214 self.loadFromString(code, root=root, filename=filename) 

1215 

1216 def loadFromString(self, code, root="config", filename=None): 

1217 """Modify this Config in place by executing the Python code in the 

1218 provided string. 

1219 

1220 Parameters 

1221 ---------- 

1222 code : `str`, `bytes`, or compiled string 

1223 Stream containing configuration override code. 

1224 root : `str`, optional 

1225 Name of the variable in file that refers to the config being 

1226 overridden. 

1227 

1228 For example, the value of root is ``"config"`` and the file 

1229 contains:: 

1230 

1231 config.myField = 5 

1232 

1233 Then this config's field ``myField`` is set to ``5``. 

1234 filename : `str`, optional 

1235 Name of the configuration file, or `None` if unknown or contained 

1236 in the stream. Used for error reporting. 

1237 

1238 See Also 

1239 -------- 

1240 lsst.pex.config.Config.load 

1241 lsst.pex.config.Config.loadFromStream 

1242 lsst.pex.config.Config.save 

1243 lsst.pex.config.Config.saveToStream 

1244 lsst.pex.config.Config.saveToString 

1245 """ 

1246 if filename is None: 1246 ↛ 1250line 1246 didn't jump to line 1250, because the condition on line 1246 was never false

1247 # try to determine the file name; a compiled string 

1248 # has attribute "co_filename", 

1249 filename = getattr(code, "co_filename", "?") 

1250 with RecordingImporter() as importer: 

1251 globals = {"__file__": filename} 

1252 local = {root: self} 

1253 exec(code, globals, local) 

1254 

1255 self._imports.update(importer.getModules()) 

1256 

1257 def save(self, filename, root="config"): 

1258 """Save a Python script to the named file, which, when loaded, 

1259 reproduces this config. 

1260 

1261 Parameters 

1262 ---------- 

1263 filename : `str` 

1264 Desination filename of this configuration. 

1265 root : `str`, optional 

1266 Name to use for the root config variable. The same value must be 

1267 used when loading (see `lsst.pex.config.Config.load`). 

1268 

1269 See Also 

1270 -------- 

1271 lsst.pex.config.Config.saveToStream 

1272 lsst.pex.config.Config.saveToString 

1273 lsst.pex.config.Config.load 

1274 lsst.pex.config.Config.loadFromStream 

1275 lsst.pex.config.Config.loadFromString 

1276 """ 

1277 d = os.path.dirname(filename) 

1278 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1279 self.saveToStream(outfile, root) 

1280 # tempfile is hardcoded to create files with mode '0600' 

1281 # for an explantion of these antics see: 

1282 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1283 umask = os.umask(0o077) 

1284 os.umask(umask) 

1285 os.chmod(outfile.name, (~umask & 0o666)) 

1286 # chmod before the move so we get quasi-atomic behavior if the 

1287 # source and dest. are on the same filesystem. 

1288 # os.rename may not work across filesystems 

1289 shutil.move(outfile.name, filename) 

1290 

1291 def saveToString(self, skipImports=False): 

1292 """Return the Python script form of this configuration as an executable 

1293 string. 

1294 

1295 Parameters 

1296 ---------- 

1297 skipImports : `bool`, optional 

1298 If `True` then do not include ``import`` statements in output, 

1299 this is to support human-oriented output from ``pipetask`` where 

1300 additional clutter is not useful. 

1301 

1302 Returns 

1303 ------- 

1304 code : `str` 

1305 A code string readable by `loadFromString`. 

1306 

1307 See Also 

1308 -------- 

1309 lsst.pex.config.Config.save 

1310 lsst.pex.config.Config.saveToStream 

1311 lsst.pex.config.Config.load 

1312 lsst.pex.config.Config.loadFromStream 

1313 lsst.pex.config.Config.loadFromString 

1314 """ 

1315 buffer = io.StringIO() 

1316 self.saveToStream(buffer, skipImports=skipImports) 

1317 return buffer.getvalue() 

1318 

1319 def saveToStream(self, outfile, root="config", skipImports=False): 

1320 """Save a configuration file to a stream, which, when loaded, 

1321 reproduces this config. 

1322 

1323 Parameters 

1324 ---------- 

1325 outfile : file-like object 

1326 Destination file object write the config into. Accepts strings not 

1327 bytes. 

1328 root 

1329 Name to use for the root config variable. The same value must be 

1330 used when loading (see `lsst.pex.config.Config.load`). 

1331 skipImports : `bool`, optional 

1332 If `True` then do not include ``import`` statements in output, 

1333 this is to support human-oriented output from ``pipetask`` where 

1334 additional clutter is not useful. 

1335 

1336 See Also 

1337 -------- 

1338 lsst.pex.config.Config.save 

1339 lsst.pex.config.Config.saveToString 

1340 lsst.pex.config.Config.load 

1341 lsst.pex.config.Config.loadFromStream 

1342 lsst.pex.config.Config.loadFromString 

1343 """ 

1344 tmp = self._name 

1345 self._rename(root) 

1346 try: 

1347 if not skipImports: 1347 ↛ 1361line 1347 didn't jump to line 1361, because the condition on line 1347 was never false

1348 self._collectImports() 

1349 # Remove self from the set, as it is handled explicitly below 

1350 self._imports.remove(self.__module__) 

1351 configType = type(self) 

1352 typeString = _typeStr(configType) 

1353 outfile.write(f"import {configType.__module__}\n") 

1354 outfile.write( 

1355 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1356 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1357 ) 

1358 for imp in sorted(self._imports): 1358 ↛ 1359line 1358 didn't jump to line 1359, because the loop on line 1358 never started

1359 if imp in sys.modules and sys.modules[imp] is not None: 

1360 outfile.write(f"import {imp}\n") 

1361 self._save(outfile) 

1362 finally: 

1363 self._rename(tmp) 

1364 

1365 def freeze(self): 

1366 """Make this config, and all subconfigs, read-only.""" 

1367 self._frozen = True 

1368 for field in self._fields.values(): 

1369 field.freeze(self) 

1370 

1371 def _save(self, outfile): 

1372 """Save this config to an open stream object. 

1373 

1374 Parameters 

1375 ---------- 

1376 outfile : file-like object 

1377 Destination file object write the config into. Accepts strings not 

1378 bytes. 

1379 """ 

1380 for field in self._fields.values(): 

1381 field.save(outfile, self) 

1382 

1383 def _collectImports(self): 

1384 """Add module containing self to the list of things to import and 

1385 then loops over all the fields in the config calling a corresponding 

1386 collect method. 

1387 

1388 The field method will call _collectImports on any 

1389 configs it may own and return the set of things to import. This 

1390 returned set will be merged with the set of imports for this config 

1391 class. 

1392 """ 

1393 self._imports.add(self.__module__) 

1394 for name, field in self._fields.items(): 

1395 field._collectImports(self, self._imports) 

1396 

1397 def toDict(self): 

1398 """Make a dictionary of field names and their values. 

1399 

1400 Returns 

1401 ------- 

1402 dict_ : `dict` 

1403 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1404 Values are `~lsst.pex.config.Field` values. 

1405 

1406 See Also 

1407 -------- 

1408 lsst.pex.config.Field.toDict 

1409 

1410 Notes 

1411 ----- 

1412 This method uses the `~lsst.pex.config.Field.toDict` method of 

1413 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1414 implement a ``toDict`` method for *this* method to work. 

1415 """ 

1416 dict_ = {} 

1417 for name, field in self._fields.items(): 

1418 dict_[name] = field.toDict(self) 

1419 return dict_ 

1420 

1421 def names(self): 

1422 """Get all the field names in the config, recursively. 

1423 

1424 Returns 

1425 ------- 

1426 names : `list` of `str` 

1427 Field names. 

1428 """ 

1429 # 

1430 # Rather than sort out the recursion all over again use the 

1431 # pre-existing saveToStream() 

1432 # 

1433 with io.StringIO() as strFd: 

1434 self.saveToStream(strFd, "config") 

1435 contents = strFd.getvalue() 

1436 strFd.close() 

1437 # 

1438 # Pull the names out of the dumped config 

1439 # 

1440 keys = [] 

1441 for line in contents.split("\n"): 

1442 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1443 continue 

1444 

1445 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1446 if mat: 

1447 keys.append(mat.group(1)) 

1448 

1449 return keys 

1450 

1451 def _rename(self, name): 

1452 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1453 

1454 Parameters 

1455 ---------- 

1456 name : `str` 

1457 New name for this config in its parent `~lsst.pex.config.Config`. 

1458 

1459 Notes 

1460 ----- 

1461 This method uses the `~lsst.pex.config.Field.rename` method of 

1462 individual `lsst.pex.config.Field` instances. 

1463 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1464 method for *this* method to work. 

1465 

1466 See Also 

1467 -------- 

1468 lsst.pex.config.Field.rename 

1469 """ 

1470 self._name = name 

1471 for field in self._fields.values(): 

1472 field.rename(self) 

1473 

1474 def validate(self): 

1475 """Validate the Config, raising an exception if invalid. 

1476 

1477 Raises 

1478 ------ 

1479 lsst.pex.config.FieldValidationError 

1480 Raised if verification fails. 

1481 

1482 Notes 

1483 ----- 

1484 The base class implementation performs type checks on all fields by 

1485 calling their `~lsst.pex.config.Field.validate` methods. 

1486 

1487 Complex single-field validation can be defined by deriving new Field 

1488 types. For convenience, some derived `lsst.pex.config.Field`-types 

1489 (`~lsst.pex.config.ConfigField` and 

1490 `~lsst.pex.config.ConfigChoiceField`) are defined in 

1491 ``lsst.pex.config`` that handle recursing into subconfigs. 

1492 

1493 Inter-field relationships should only be checked in derived 

1494 `~lsst.pex.config.Config` classes after calling this method, and base 

1495 validation is complete. 

1496 """ 

1497 for field in self._fields.values(): 

1498 field.validate(self) 

1499 

1500 def formatHistory(self, name, **kwargs): 

1501 """Format a configuration field's history to a human-readable format. 

1502 

1503 Parameters 

1504 ---------- 

1505 name : `str` 

1506 Name of a `~lsst.pex.config.Field` in this config. 

1507 kwargs 

1508 Keyword arguments passed to `lsst.pex.config.history.format`. 

1509 

1510 Returns 

1511 ------- 

1512 history : `str` 

1513 A string containing the formatted history. 

1514 

1515 See Also 

1516 -------- 

1517 lsst.pex.config.history.format 

1518 """ 

1519 import lsst.pex.config.history as pexHist 

1520 

1521 return pexHist.format(self, name, **kwargs) 

1522 

1523 history = property(lambda x: x._history) 1523 ↛ exitline 1523 didn't run the lambda on line 1523

1524 """Read-only history. 

1525 """ 

1526 

1527 def __setattr__(self, attr, value, at=None, label="assignment"): 

1528 """Set an attribute (such as a field's value). 

1529 

1530 Notes 

1531 ----- 

1532 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1533 locked such that no additional attributes nor properties may be added 

1534 to them dynamically. 

1535 

1536 Although this is not the standard Python behavior, it helps to protect 

1537 users from accidentally mispelling a field name, or trying to set a 

1538 non-existent field. 

1539 """ 

1540 if attr in self._fields: 

1541 if self._fields[attr].deprecated is not None: 1541 ↛ 1542line 1541 didn't jump to line 1542, because the condition on line 1541 was never true

1542 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1543 warnings.warn( 

1544 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1545 FutureWarning, 

1546 stacklevel=2, 

1547 ) 

1548 if at is None: 1548 ↛ 1551line 1548 didn't jump to line 1551, because the condition on line 1548 was never false

1549 at = getCallStack() 

1550 # This allows Field descriptors to work. 

1551 self._fields[attr].__set__(self, value, at=at, label=label) 

1552 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1552 ↛ 1554line 1552 didn't jump to line 1554, because the condition on line 1552 was never true

1553 # This allows properties and other non-Field descriptors to work. 

1554 return object.__setattr__(self, attr, value) 

1555 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1555 ↛ 1560line 1555 didn't jump to line 1560, because the condition on line 1555 was never false

1556 # This allows specific private attributes to work. 

1557 self.__dict__[attr] = value 

1558 else: 

1559 # We throw everything else. 

1560 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}") 

1561 

1562 def __delattr__(self, attr, at=None, label="deletion"): 

1563 if attr in self._fields: 

1564 if at is None: 

1565 at = getCallStack() 

1566 self._fields[attr].__delete__(self, at=at, label=label) 

1567 else: 

1568 object.__delattr__(self, attr) 

1569 

1570 def __eq__(self, other): 

1571 if type(other) == type(self): 1571 ↛ 1572line 1571 didn't jump to line 1572, because the condition on line 1571 was never true

1572 for name in self._fields: 

1573 thisValue = getattr(self, name) 

1574 otherValue = getattr(other, name) 

1575 if isinstance(thisValue, float) and math.isnan(thisValue): 

1576 if not math.isnan(otherValue): 

1577 return False 

1578 elif thisValue != otherValue: 

1579 return False 

1580 return True 

1581 return False 

1582 

1583 def __ne__(self, other): 

1584 return not self.__eq__(other) 

1585 

1586 def __str__(self): 

1587 return str(self.toDict()) 

1588 

1589 def __repr__(self): 

1590 return "{}({})".format( 

1591 _typeStr(self), 

1592 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None), 

1593 ) 

1594 

1595 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1596 """Compare this configuration to another `~lsst.pex.config.Config` for 

1597 equality. 

1598 

1599 Parameters 

1600 ---------- 

1601 other : `lsst.pex.config.Config` 

1602 Other `~lsst.pex.config.Config` object to compare against this 

1603 config. 

1604 shortcut : `bool`, optional 

1605 If `True`, return as soon as an inequality is found. Default is 

1606 `True`. 

1607 rtol : `float`, optional 

1608 Relative tolerance for floating point comparisons. 

1609 atol : `float`, optional 

1610 Absolute tolerance for floating point comparisons. 

1611 output : callable, optional 

1612 A callable that takes a string, used (possibly repeatedly) to 

1613 report inequalities. 

1614 

1615 Returns 

1616 ------- 

1617 isEqual : `bool` 

1618 `True` when the two `lsst.pex.config.Config` instances are equal. 

1619 `False` if there is an inequality. 

1620 

1621 See Also 

1622 -------- 

1623 lsst.pex.config.compareConfigs 

1624 

1625 Notes 

1626 ----- 

1627 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1628 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1629 are not considered by this method. 

1630 

1631 Floating point comparisons are performed by `numpy.allclose`. 

1632 """ 

1633 name1 = self._name if self._name is not None else "config" 

1634 name2 = other._name if other._name is not None else "config" 

1635 name = getComparisonName(name1, name2) 

1636 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1637 

1638 @classmethod 

1639 def __init_subclass__(cls, **kwargs): 

1640 """Run initialization for every subclass. 

1641 

1642 Specifically registers the subclass with a YAML representer 

1643 and YAML constructor (if pyyaml is available) 

1644 """ 

1645 super().__init_subclass__(**kwargs) 

1646 

1647 if not yaml: 1647 ↛ 1648line 1647 didn't jump to line 1648, because the condition on line 1647 was never true

1648 return 

1649 

1650 yaml.add_representer(cls, _yaml_config_representer) 

1651 

1652 @classmethod 

1653 def _fromPython(cls, config_py): 

1654 """Instantiate a `Config`-subclass from serialized Python form. 

1655 

1656 Parameters 

1657 ---------- 

1658 config_py : `str` 

1659 A serialized form of the Config as created by 

1660 `Config.saveToStream`. 

1661 

1662 Returns 

1663 ------- 

1664 config : `Config` 

1665 Reconstructed `Config` instant. 

1666 """ 

1667 cls = _classFromPython(config_py) 

1668 return unreduceConfig(cls, config_py) 

1669 

1670 

1671def _classFromPython(config_py): 

1672 """Return the Config subclass required by this Config serialization. 

1673 

1674 Parameters 

1675 ---------- 

1676 config_py : `str` 

1677 A serialized form of the Config as created by 

1678 `Config.saveToStream`. 

1679 

1680 Returns 

1681 ------- 

1682 cls : `type` 

1683 The `Config` subclass associated with this config. 

1684 """ 

1685 # standard serialization has the form: 

1686 # import config.class 

1687 # assert type(config)==config.class.Config, ... 

1688 # We want to parse these two lines so we can get the class itself 

1689 

1690 # Do a single regex to avoid large string copies when splitting a 

1691 # large config into separate lines. 

1692 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1693 

1694 if not matches: 

1695 first_line, second_line, _ = config_py.split("\n", 2) 

1696 raise ValueError( 

1697 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}" 

1698 ) 

1699 

1700 module_name = matches.group(1) 

1701 module = importlib.import_module(module_name) 

1702 

1703 # Second line 

1704 full_name = matches.group(2) 

1705 

1706 # Remove the module name from the full name 

1707 if not full_name.startswith(module_name): 

1708 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1709 

1710 # if module name is a.b.c and full name is a.b.c.d.E then 

1711 # we need to remove a.b.c. and iterate over the remainder 

1712 # The +1 is for the extra dot after a.b.c 

1713 remainder = full_name[len(module_name) + 1 :] 

1714 components = remainder.split(".") 

1715 pytype = module 

1716 for component in components: 

1717 pytype = getattr(pytype, component) 

1718 return pytype 

1719 

1720 

1721def unreduceConfig(cls, stream): 

1722 """Create a `~lsst.pex.config.Config` from a stream. 

1723 

1724 Parameters 

1725 ---------- 

1726 cls : `lsst.pex.config.Config`-type 

1727 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1728 with configurations in the ``stream``. 

1729 stream : file-like object, `str`, or compiled string 

1730 Stream containing configuration override code. 

1731 

1732 Returns 

1733 ------- 

1734 config : `lsst.pex.config.Config` 

1735 Config instance. 

1736 

1737 See Also 

1738 -------- 

1739 lsst.pex.config.Config.loadFromStream 

1740 """ 

1741 config = cls() 

1742 config.loadFromStream(stream) 

1743 return config