Coverage for python/lsst/pex/config/config.py: 58%

462 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-01 12:22 +0000

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from collections.abc import Mapping 

49from typing import Any, ForwardRef, Generic, TypeVar, cast, overload 

50 

51try: 

52 from types import GenericAlias 

53except ImportError: 

54 # cover python 3.8 usage 

55 GenericAlias = type(Mapping[int, int]) 

56 

57# if YAML is not available that's fine and we simply don't register 

58# the yaml representer since we know it won't be used. 

59try: 

60 import yaml 

61except ImportError: 

62 yaml = None 

63 

64from .callStack import getCallStack, getStackFrame 

65from .comparison import compareConfigs, compareScalars, getComparisonName 

66 

67if yaml: 67 ↛ 78line 67 didn't jump to line 78, because the condition on line 67 was never false

68 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

69 

70 try: 

71 # CLoader is not always available 

72 from yaml import CLoader 

73 

74 YamlLoaders += (CLoader,) 

75 except ImportError: 

76 pass 

77else: 

78 YamlLoaders = () 

79 doImport = None 

80 

81 

82class _PexConfigGenericAlias(GenericAlias): 

83 """A Subclass of python's GenericAlias used in defining and instantiating 

84 Generics. 

85 

86 This class differs from `types.GenericAlias` in that it calls a method 

87 named _parseTypingArgs defined on Fields. This method gives Field and its 

88 subclasses an opportunity to transform type parameters into class key word 

89 arguments. Code authors do not need to implement any returns of this object 

90 directly, and instead only need implement _parseTypingArgs, if a Field 

91 subclass differs from the base class implementation. 

92 

93 This class is intended to be an implementation detail, returned from a 

94 Field's `__class_getitem__` method. 

95 """ 

96 

97 def __call__(self, *args: Any, **kwds: Any) -> Any: 

98 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

99 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

100 

101 

102FieldTypeVar = TypeVar("FieldTypeVar") 

103 

104 

105class UnexpectedProxyUsageError(TypeError): 

106 """Exception raised when a proxy class is used in a context that suggests 

107 it should have already been converted to the thing it proxies. 

108 """ 

109 

110 

111def _joinNamePath(prefix=None, name=None, index=None): 

112 """Generate nested configuration names.""" 

113 if not prefix and not name: 113 ↛ 114line 113 didn't jump to line 114, because the condition on line 113 was never true

114 raise ValueError("Invalid name: cannot be None") 

115 elif not name: 115 ↛ 116line 115 didn't jump to line 116, because the condition on line 115 was never true

116 name = prefix 

117 elif prefix and name: 117 ↛ 120line 117 didn't jump to line 120, because the condition on line 117 was never false

118 name = prefix + "." + name 

119 

120 if index is not None: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 return f"{name}[{index!r}]" 

122 else: 

123 return name 

124 

125 

126def _autocast(x, dtype): 

127 """Cast a value to a type, if appropriate. 

128 

129 Parameters 

130 ---------- 

131 x : object 

132 A value. 

133 dtype : tpye 

134 Data type, such as `float`, `int`, or `str`. 

135 

136 Returns 

137 ------- 

138 values : object 

139 If appropriate, the returned value is ``x`` cast to the given type 

140 ``dtype``. If the cast cannot be performed the original value of 

141 ``x`` is returned. 

142 """ 

143 if dtype == float and isinstance(x, int): 

144 return float(x) 

145 return x 

146 

147 

148def _typeStr(x): 

149 """Generate a fully-qualified type name. 

150 

151 Returns 

152 ------- 

153 `str` 

154 Fully-qualified type name. 

155 

156 Notes 

157 ----- 

158 This function is used primarily for writing config files to be executed 

159 later upon with the 'load' function. 

160 """ 

161 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

162 xtype = x 

163 else: 

164 xtype = type(x) 

165 if xtype.__module__ == "builtins": 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true

166 return xtype.__name__ 

167 else: 

168 return f"{xtype.__module__}.{xtype.__name__}" 

169 

170 

171if yaml: 171 ↛ 204line 171 didn't jump to line 204, because the condition on line 171 was never false

172 

173 def _yaml_config_representer(dumper, data): 

174 """Represent a Config object in a form suitable for YAML. 

175 

176 Stores the serialized stream as a scalar block string. 

177 """ 

178 stream = io.StringIO() 

179 data.saveToStream(stream) 

180 config_py = stream.getvalue() 

181 

182 # Strip multiple newlines from the end of the config 

183 # This simplifies the YAML to use | and not |+ 

184 config_py = config_py.rstrip() + "\n" 

185 

186 # Trailing spaces force pyyaml to use non-block form. 

187 # Remove the trailing spaces so it has no choice 

188 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

189 

190 # Store the Python as a simple scalar 

191 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

192 

193 def _yaml_config_constructor(loader, node): 

194 """Construct a config from YAML.""" 

195 config_py = loader.construct_scalar(node) 

196 return Config._fromPython(config_py) 

197 

198 # Register a generic constructor for Config and all subclasses 

199 # Need to register for all the loaders we would like to use 

200 for loader in YamlLoaders: 

201 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

202 

203 

204class ConfigMeta(type): 

205 """A metaclass for `lsst.pex.config.Config`. 

206 

207 Parameters 

208 ---------- 

209 name : `str` 

210 Name to use for class. 

211 bases : `~collections.abc.Iterable` 

212 Base classes. 

213 dict_ : `dict` 

214 Additional parameters. 

215 

216 Notes 

217 ----- 

218 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

219 class attributes as a class attribute called ``_fields``, and adds 

220 the name of each field as an instance variable of the field itself (so you 

221 don't have to pass the name of the field to the field constructor). 

222 """ 

223 

224 def __init__(cls, name, bases, dict_): 

225 type.__init__(cls, name, bases, dict_) 

226 cls._fields = {} 

227 cls._source = getStackFrame() 

228 

229 def getFields(classtype): 

230 fields = {} 

231 bases = list(classtype.__bases__) 

232 bases.reverse() 

233 for b in bases: 

234 fields.update(getFields(b)) 

235 

236 for k, v in classtype.__dict__.items(): 

237 if isinstance(v, Field): 

238 fields[k] = v 

239 return fields 

240 

241 fields = getFields(cls) 

242 for k, v in fields.items(): 

243 setattr(cls, k, copy.deepcopy(v)) 

244 

245 def __setattr__(cls, name, value): 

246 if isinstance(value, Field): 

247 value.name = name 

248 cls._fields[name] = value 

249 type.__setattr__(cls, name, value) 

250 

251 

252class FieldValidationError(ValueError): 

253 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

254 particular ``~lsst.pex.config.Config``. 

255 

256 Parameters 

257 ---------- 

258 field : `lsst.pex.config.Field` 

259 The field that was not valid. 

260 config : `lsst.pex.config.Config` 

261 The config containing the invalid field. 

262 msg : `str` 

263 Text describing why the field was not valid. 

264 """ 

265 

266 def __init__(self, field, config, msg): 

267 self.fieldType = type(field) 

268 """Type of the `~lsst.pex.config.Field` that incurred the error. 

269 """ 

270 

271 self.fieldName = field.name 

272 """Name of the `~lsst.pex.config.Field` instance that incurred the 

273 error (`str`). 

274 

275 See also 

276 -------- 

277 lsst.pex.config.Field.name 

278 """ 

279 

280 self.fullname = _joinNamePath(config._name, field.name) 

281 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

282 (`str`). 

283 """ 

284 

285 self.history = config.history.setdefault(field.name, []) 

286 """Full history of all changes to the `~lsst.pex.config.Field` 

287 instance. 

288 """ 

289 

290 self.fieldSource = field.source 

291 """File and line number of the `~lsst.pex.config.Field` definition. 

292 """ 

293 

294 self.configSource = config._source 

295 error = ( 

296 "%s '%s' failed validation: %s\n" 

297 "For more information see the Field definition at:\n%s" 

298 " and the Config definition at:\n%s" 

299 % ( 

300 self.fieldType.__name__, 

301 self.fullname, 

302 msg, 

303 self.fieldSource.format(), 

304 self.configSource.format(), 

305 ) 

306 ) 

307 super().__init__(error) 

308 

309 

310class Field(Generic[FieldTypeVar]): 

311 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

312 `complex`, `bool`, and `str` data types. 

313 

314 Parameters 

315 ---------- 

316 doc : `str` 

317 A description of the field for users. 

318 dtype : type, optional 

319 The field's data type. ``Field`` only supports basic data types: 

320 `int`, `float`, `complex`, `bool`, and `str`. See 

321 `Field.supportedTypes`. Optional if supplied as a typing argument to 

322 the class. 

323 default : object, optional 

324 The field's default value. 

325 check : callable, optional 

326 A callable that is called with the field's value. This callable should 

327 return `False` if the value is invalid. More complex inter-field 

328 validation can be written as part of the 

329 `lsst.pex.config.Config.validate` method. 

330 optional : `bool`, optional 

331 This sets whether the field is considered optional, and therefore 

332 doesn't need to be set by the user. When `False`, 

333 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

334 deprecated : None or `str`, optional 

335 A description of why this Field is deprecated, including removal date. 

336 If not None, the string is appended to the docstring for this Field. 

337 

338 Raises 

339 ------ 

340 ValueError 

341 Raised when the ``dtype`` parameter is not one of the supported types 

342 (see `Field.supportedTypes`). 

343 

344 See Also 

345 -------- 

346 ChoiceField 

347 ConfigChoiceField 

348 ConfigDictField 

349 ConfigField 

350 ConfigurableField 

351 DictField 

352 ListField 

353 RangeField 

354 RegistryField 

355 

356 Notes 

357 ----- 

358 ``Field`` instances (including those of any subclass of ``Field``) are used 

359 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

360 example, below). ``Field`` attributes work like the `property` attributes 

361 of classes that implement custom setters and getters. `Field` attributes 

362 belong to the class, but operate on the instance. Formally speaking, 

363 `Field` attributes are `descriptors 

364 <https://docs.python.org/3/howto/descriptor.html>`_. 

365 

366 When you access a `Field` attribute on a `Config` instance, you don't 

367 get the `Field` instance itself. Instead, you get the value of that field, 

368 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

369 container type (like a `lsst.pex.config.List`) depending on the field's 

370 type. See the example, below. 

371 

372 Fields can be annotated with a type similar to other python classes (python 

373 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

374 See the name field in the Config example below for an example of this. 

375 Unlike most other uses in python, this has an effect at type checking *and* 

376 runtime. If the type is specified with a class annotation, it will be used 

377 as the value of the ``dtype`` in the ``Field`` and there is no need to 

378 specify it as an argument during instantiation. 

379 

380 There are Some notes on dtype through type annotation syntax. Type 

381 annotation syntax supports supplying the argument as a string of a type 

382 name. i.e. "float", but this cannot be used to resolve circular references. 

383 Type annotation syntax can be used on an identifier in addition to Class 

384 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

385 ``someField = Field[str](doc="some doc"). However, this syntax is only 

386 useful for annotating the type of the identifier (i.e. variable in previous 

387 example) and does nothing for assigning the dtype of the ``Field``. 

388 

389 Examples 

390 -------- 

391 Instances of ``Field`` should be used as class attributes of 

392 `lsst.pex.config.Config` subclasses: 

393 

394 >>> from lsst.pex.config import Config, Field 

395 >>> class Example(Config): 

396 ... myInt = Field("An integer field.", int, default=0) 

397 ... name = Field[str](doc="A string Field") 

398 ... 

399 >>> print(config.myInt) 

400 0 

401 >>> config.myInt = 5 

402 >>> print(config.myInt) 

403 5 

404 """ 

405 

406 name: str 

407 """Identifier (variable name) used to refer to a Field within a Config 

408 Class. 

409 """ 

410 

411 supportedTypes = {str, bool, float, int, complex} 

412 """Supported data types for field values (`set` of types). 

413 """ 

414 

415 @staticmethod 

416 def _parseTypingArgs( 

417 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any] 

418 ) -> Mapping[str, Any]: 

419 """Parse type annotations into keyword constructor arguments. 

420 

421 This is a special private method that interprets type arguments (i.e. 

422 Field[str]) into keyword arguments to be passed on to the constructor. 

423 

424 Subclasses of Field can implement this method to customize how they 

425 handle turning type parameters into keyword arguments (see DictField 

426 for an example) 

427 

428 Parameters 

429 ---------- 

430 params : `tuple` of `type` or `tuple` of str 

431 Parameters passed to the type annotation. These will either be 

432 types or strings. Strings are to interpreted as forward references 

433 and will be treated as such. 

434 kwds : `MutableMapping` with keys of `str` and values of `Any` 

435 These are the user supplied keywords that are to be passed to the 

436 Field constructor. 

437 

438 Returns 

439 ------- 

440 kwds : `MutableMapping` with keys of `str` and values of `Any` 

441 The mapping of keywords that will be passed onto the constructor 

442 of the Field. Should be filled in with any information gleaned 

443 from the input parameters. 

444 

445 Raises 

446 ------ 

447 ValueError 

448 Raised if params is of incorrect length. 

449 Raised if a forward reference could not be resolved 

450 Raised if there is a conflict between params and values in kwds 

451 """ 

452 if len(params) > 1: 

453 raise ValueError("Only single type parameters are supported") 

454 unpackedParams = params[0] 

455 if isinstance(unpackedParams, str): 

456 _typ = ForwardRef(unpackedParams) 

457 # type ignore below because typeshed seems to be wrong. It 

458 # indicates there are only 2 args, as it was in python 3.8, but 

459 # 3.9+ takes 3 args. Attempt in old style and new style to 

460 # work with both. 

461 try: 

462 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

463 except TypeError: 

464 # python 3.8 path 

465 result = _typ._evaluate(globals(), locals()) 

466 if result is None: 

467 raise ValueError("Could not deduce type from input") 

468 unpackedParams = cast(type, result) 

469 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

470 raise ValueError("Conflicting definition for dtype") 

471 elif "dtype" not in kwds: 

472 kwds = {**kwds, **{"dtype": unpackedParams}} 

473 return kwds 

474 

475 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef): 

476 return _PexConfigGenericAlias(cls, params) 

477 

478 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

479 if dtype is None: 479 ↛ 480line 479 didn't jump to line 480, because the condition on line 479 was never true

480 raise ValueError( 

481 "dtype must either be supplied as an argument or as a type argument to the class" 

482 ) 

483 if dtype not in self.supportedTypes: 483 ↛ 484line 483 didn't jump to line 484, because the condition on line 483 was never true

484 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

485 

486 source = getStackFrame() 

487 self._setup( 

488 doc=doc, 

489 dtype=dtype, 

490 default=default, 

491 check=check, 

492 optional=optional, 

493 source=source, 

494 deprecated=deprecated, 

495 ) 

496 

497 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

498 """Set attributes, usually during initialization.""" 

499 self.dtype = dtype 

500 """Data type for the field. 

501 """ 

502 

503 if not doc: 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 raise ValueError("Docstring is empty.") 

505 

506 # append the deprecation message to the docstring. 

507 if deprecated is not None: 

508 doc = f"{doc} Deprecated: {deprecated}" 

509 self.doc = doc 

510 """A description of the field (`str`). 

511 """ 

512 

513 self.deprecated = deprecated 

514 """If not None, a description of why this field is deprecated (`str`). 

515 """ 

516 

517 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

518 if optional or default is not None: 

519 self.__doc__ += f", default ``{default!r}``" 

520 self.__doc__ += ")" 

521 

522 self.default = default 

523 """Default value for this field. 

524 """ 

525 

526 self.check = check 

527 """A user-defined function that validates the value of the field. 

528 """ 

529 

530 self.optional = optional 

531 """Flag that determines if the field is required to be set (`bool`). 

532 

533 When `False`, `lsst.pex.config.Config.validate` will fail if the 

534 field's value is `None`. 

535 """ 

536 

537 self.source = source 

538 """The stack frame where this field is defined (`list` of 

539 `~lsst.pex.config.callStack.StackFrame`). 

540 """ 

541 

542 def rename(self, instance): 

543 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use 

544 only). 

545 

546 Parameters 

547 ---------- 

548 instance : `lsst.pex.config.Config` 

549 The config instance that contains this field. 

550 

551 Notes 

552 ----- 

553 This method is invoked by the `lsst.pex.config.Config` object that 

554 contains this field and should not be called directly. 

555 

556 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

557 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should 

558 rename each subconfig with the full field name as generated by 

559 `lsst.pex.config.config._joinNamePath`. 

560 """ 

561 pass 

562 

563 def validate(self, instance): 

564 """Validate the field (for internal use only). 

565 

566 Parameters 

567 ---------- 

568 instance : `lsst.pex.config.Config` 

569 The config instance that contains this field. 

570 

571 Raises 

572 ------ 

573 lsst.pex.config.FieldValidationError 

574 Raised if verification fails. 

575 

576 Notes 

577 ----- 

578 This method provides basic validation: 

579 

580 - Ensures that the value is not `None` if the field is not optional. 

581 - Ensures type correctness. 

582 - Ensures that the user-provided ``check`` function is valid. 

583 

584 Most `~lsst.pex.config.Field` subclasses should call 

585 `lsst.pex.config.Field.validate` if they re-implement 

586 `~lsst.pex.config.Field.validate`. 

587 """ 

588 value = self.__get__(instance) 

589 if not self.optional and value is None: 

590 raise FieldValidationError(self, instance, "Required value cannot be None") 

591 

592 def freeze(self, instance): 

593 """Make this field read-only (for internal use only). 

594 

595 Parameters 

596 ---------- 

597 instance : `lsst.pex.config.Config` 

598 The config instance that contains this field. 

599 

600 Notes 

601 ----- 

602 Freezing is only relevant for fields that hold subconfigs. Fields which 

603 hold subconfigs should freeze each subconfig. 

604 

605 **Subclasses should implement this method.** 

606 """ 

607 pass 

608 

609 def _validateValue(self, value): 

610 """Validate a value. 

611 

612 Parameters 

613 ---------- 

614 value : object 

615 The value being validated. 

616 

617 Raises 

618 ------ 

619 TypeError 

620 Raised if the value's type is incompatible with the field's 

621 ``dtype``. 

622 ValueError 

623 Raised if the value is rejected by the ``check`` method. 

624 """ 

625 if value is None: 625 ↛ 626line 625 didn't jump to line 626, because the condition on line 625 was never true

626 return 

627 

628 if not isinstance(value, self.dtype): 628 ↛ 629line 628 didn't jump to line 629, because the condition on line 628 was never true

629 msg = "Value {} is of incorrect type {}. Expected type {}".format( 

630 value, 

631 _typeStr(value), 

632 _typeStr(self.dtype), 

633 ) 

634 raise TypeError(msg) 

635 if self.check is not None and not self.check(value): 635 ↛ 636line 635 didn't jump to line 636, because the condition on line 635 was never true

636 msg = "Value %s is not a valid value" % str(value) 

637 raise ValueError(msg) 

638 

639 def _collectImports(self, instance, imports): 

640 """Call the _collectImports method on all config 

641 objects the field may own, and union them with the supplied imports 

642 set. 

643 

644 Parameters 

645 ---------- 

646 instance : instance or subclass of `lsst.pex.config.Config` 

647 A config object that has this field defined on it 

648 imports : `set` 

649 Set of python modules that need imported after persistence 

650 """ 

651 pass 

652 

653 def save(self, outfile, instance): 

654 """Save this field to a file (for internal use only). 

655 

656 Parameters 

657 ---------- 

658 outfile : file-like object 

659 A writeable field handle. 

660 instance : `~lsst.pex.config.Config` 

661 The `~lsst.pex.config.Config` instance that contains this field. 

662 

663 Notes 

664 ----- 

665 This method is invoked by the `~lsst.pex.config.Config` object that 

666 contains this field and should not be called directly. 

667 

668 The output consists of the documentation string 

669 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

670 line is formatted as an assignment: ``{fullname}={value}``. 

671 

672 This output can be executed with Python. 

673 """ 

674 value = self.__get__(instance) 

675 fullname = _joinNamePath(instance._name, self.name) 

676 

677 if self.deprecated and value == self.default: 677 ↛ 678line 677 didn't jump to line 678, because the condition on line 677 was never true

678 return 

679 

680 # write full documentation string as comment lines 

681 # (i.e. first character is #) 

682 doc = "# " + str(self.doc).replace("\n", "\n# ") 

683 if isinstance(value, float) and not math.isfinite(value): 683 ↛ 685line 683 didn't jump to line 685, because the condition on line 683 was never true

684 # non-finite numbers need special care 

685 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n") 

686 else: 

687 outfile.write(f"{doc}\n{fullname}={value!r}\n\n") 

688 

689 def toDict(self, instance): 

690 """Convert the field value so that it can be set as the value of an 

691 item in a `dict` (for internal use only). 

692 

693 Parameters 

694 ---------- 

695 instance : `~lsst.pex.config.Config` 

696 The `~lsst.pex.config.Config` that contains this field. 

697 

698 Returns 

699 ------- 

700 value : object 

701 The field's value. See *Notes*. 

702 

703 Notes 

704 ----- 

705 This method invoked by the owning `~lsst.pex.config.Config` object and 

706 should not be called directly. 

707 

708 Simple values are passed through. Complex data structures must be 

709 manipulated. For example, a `~lsst.pex.config.Field` holding a 

710 subconfig should, instead of the subconfig object, return a `dict` 

711 where the keys are the field names in the subconfig, and the values are 

712 the field values in the subconfig. 

713 """ 

714 return self.__get__(instance) 

715 

716 @overload 

717 def __get__( 

718 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

719 ) -> Field[FieldTypeVar]: 

720 ... 

721 

722 @overload 

723 def __get__( 

724 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default" 

725 ) -> FieldTypeVar: 

726 ... 

727 

728 def __get__(self, instance, owner=None, at=None, label="default"): 

729 """Define how attribute access should occur on the Config instance 

730 This is invoked by the owning config object and should not be called 

731 directly. 

732 

733 When the field attribute is accessed on a Config class object, it 

734 returns the field object itself in order to allow inspection of 

735 Config classes. 

736 

737 When the field attribute is access on a config instance, the actual 

738 value described by the field (and held by the Config instance) is 

739 returned. 

740 """ 

741 if instance is None: 741 ↛ 742line 741 didn't jump to line 742, because the condition on line 741 was never true

742 return self 

743 else: 

744 # try statements are almost free in python if they succeed 

745 try: 

746 return instance._storage[self.name] 

747 except AttributeError: 

748 if not isinstance(instance, Config): 

749 return self 

750 else: 

751 raise AttributeError( 

752 f"Config {instance} is missing _storage attribute, likely incorrectly initialized" 

753 ) 

754 

755 def __set__( 

756 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment" 

757 ) -> None: 

758 """Set an attribute on the config instance. 

759 

760 Parameters 

761 ---------- 

762 instance : `lsst.pex.config.Config` 

763 The config instance that contains this field. 

764 value : obj 

765 Value to set on this field. 

766 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\ 

767 optional 

768 The call stack (created by 

769 `lsst.pex.config.callStack.getCallStack`). 

770 label : `str`, optional 

771 Event label for the history. 

772 

773 Notes 

774 ----- 

775 This method is invoked by the owning `lsst.pex.config.Config` object 

776 and should not be called directly. 

777 

778 Derived `~lsst.pex.config.Field` classes may need to override the 

779 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

780 should follow the following rules: 

781 

782 - Do not allow modification of frozen configs. 

783 - Validate the new value **before** modifying the field. Except if the 

784 new value is `None`. `None` is special and no attempt should be made 

785 to validate it until `lsst.pex.config.Config.validate` is called. 

786 - Do not modify the `~lsst.pex.config.Config` instance to contain 

787 invalid values. 

788 - If the field is modified, update the history of the 

789 `lsst.pex.config.field.Field` to reflect the changes. 

790 

791 In order to decrease the need to implement this method in derived 

792 `~lsst.pex.config.Field` types, value validation is performed in the 

793 `lsst.pex.config.Field._validateValue`. If only the validation step 

794 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

795 implement `lsst.pex.config.Field._validateValue` than to reimplement 

796 ``__set__``. More complicated behavior, however, may require 

797 reimplementation. 

798 """ 

799 if instance._frozen: 799 ↛ 800line 799 didn't jump to line 800, because the condition on line 799 was never true

800 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

801 

802 history = instance._history.setdefault(self.name, []) 

803 if value is not None: 803 ↛ 810line 803 didn't jump to line 810, because the condition on line 803 was never false

804 value = _autocast(value, self.dtype) 

805 try: 

806 self._validateValue(value) 

807 except BaseException as e: 

808 raise FieldValidationError(self, instance, str(e)) 

809 

810 instance._storage[self.name] = value 

811 if at is None: 811 ↛ 812line 811 didn't jump to line 812, because the condition on line 811 was never true

812 at = getCallStack() 

813 history.append((value, at, label)) 

814 

815 def __delete__(self, instance, at=None, label="deletion"): 

816 """Delete an attribute from a `lsst.pex.config.Config` instance. 

817 

818 Parameters 

819 ---------- 

820 instance : `lsst.pex.config.Config` 

821 The config instance that contains this field. 

822 at : `list` of `lsst.pex.config.callStack.StackFrame` 

823 The call stack (created by 

824 `lsst.pex.config.callStack.getCallStack`). 

825 label : `str`, optional 

826 Event label for the history. 

827 

828 Notes 

829 ----- 

830 This is invoked by the owning `~lsst.pex.config.Config` object and 

831 should not be called directly. 

832 """ 

833 if at is None: 

834 at = getCallStack() 

835 self.__set__(instance, None, at=at, label=label) 

836 

837 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

838 """Compare a field (named `Field.name`) in two 

839 `~lsst.pex.config.Config` instances for equality. 

840 

841 Parameters 

842 ---------- 

843 instance1 : `lsst.pex.config.Config` 

844 Left-hand side `Config` instance to compare. 

845 instance2 : `lsst.pex.config.Config` 

846 Right-hand side `Config` instance to compare. 

847 shortcut : `bool`, optional 

848 **Unused.** 

849 rtol : `float`, optional 

850 Relative tolerance for floating point comparisons. 

851 atol : `float`, optional 

852 Absolute tolerance for floating point comparisons. 

853 output : callable, optional 

854 A callable that takes a string, used (possibly repeatedly) to 

855 report inequalities. 

856 

857 Notes 

858 ----- 

859 This method must be overridden by more complex `Field` subclasses. 

860 

861 See Also 

862 -------- 

863 lsst.pex.config.compareScalars 

864 """ 

865 v1 = getattr(instance1, self.name) 

866 v2 = getattr(instance2, self.name) 

867 name = getComparisonName( 

868 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

869 ) 

870 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

871 

872 

873class RecordingImporter: 

874 """Importer (for `sys.meta_path`) that records which modules are being 

875 imported. 

876 

877 *This class does not do any importing itself.* 

878 

879 Examples 

880 -------- 

881 Use this class as a context manager to ensure it is properly uninstalled 

882 when done: 

883 

884 >>> with RecordingImporter() as importer: 

885 ... # import stuff 

886 ... import numpy as np 

887 ... print("Imported: " + importer.getModules()) 

888 """ 

889 

890 def __init__(self): 

891 self._modules = set() 

892 

893 def __enter__(self): 

894 self.origMetaPath = sys.meta_path 

895 sys.meta_path = [self] + sys.meta_path # type: ignore 

896 return self 

897 

898 def __exit__(self, *args): 

899 self.uninstall() 

900 return False # Don't suppress exceptions 

901 

902 def uninstall(self): 

903 """Uninstall the importer.""" 

904 sys.meta_path = self.origMetaPath 

905 

906 def find_spec(self, fullname, path, target=None): 

907 """Find a module. 

908 

909 Called as part of the ``import`` chain of events. 

910 

911 Parameters 

912 ---------- 

913 fullname : `str` 

914 Name of module. 

915 path : `list` [`str`] 

916 Search path. Unused. 

917 target : `~typing.Any`, optional 

918 Unused. 

919 """ 

920 self._modules.add(fullname) 

921 # Return None because we don't do any importing. 

922 return None 

923 

924 def getModules(self): 

925 """Get the set of modules that were imported. 

926 

927 Returns 

928 ------- 

929 modules : `set` of `str` 

930 Set of imported module names. 

931 """ 

932 return self._modules 

933 

934 

935# type ignore because type checker thinks ConfigMeta is Generic when it is not 

936class Config(metaclass=ConfigMeta): # type: ignore 

937 """Base class for configuration (*config*) objects. 

938 

939 Notes 

940 ----- 

941 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

942 instances as class attributes. These are used to define most of the base 

943 class behavior. 

944 

945 ``Config`` implements a mapping API that provides many `dict`-like methods, 

946 such as `keys`, `values`, and `items`. ``Config`` instances also support 

947 the ``in`` operator to test if a field is in the config. Unlike a `dict`, 

948 ``Config`` classes are not subscriptable. Instead, access individual 

949 fields as attributes of the configuration instance. 

950 

951 Examples 

952 -------- 

953 Config classes are subclasses of ``Config`` that have 

954 `~lsst.pex.config.Field` instances (or instances of 

955 `~lsst.pex.config.Field` subclasses) as class attributes: 

956 

957 >>> from lsst.pex.config import Config, Field, ListField 

958 >>> class DemoConfig(Config): 

959 ... intField = Field(doc="An integer field", dtype=int, default=42) 

960 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

961 ... default=['coffee', 'green tea', 'water']) 

962 ... 

963 >>> config = DemoConfig() 

964 

965 Configs support many `dict`-like APIs: 

966 

967 >>> config.keys() 

968 ['intField', 'listField'] 

969 >>> 'intField' in config 

970 True 

971 

972 Individual fields can be accessed as attributes of the configuration: 

973 

974 >>> config.intField 

975 42 

976 >>> config.listField.append('earl grey tea') 

977 >>> print(config.listField) 

978 ['coffee', 'green tea', 'water', 'earl grey tea'] 

979 """ 

980 

981 _storage: dict[str, Any] 

982 _fields: dict[str, Field] 

983 _history: dict[str, list[Any]] 

984 _imports: set[Any] 

985 

986 def __iter__(self): 

987 """Iterate over fields.""" 

988 return self._fields.__iter__() 

989 

990 def keys(self): 

991 """Get field names. 

992 

993 Returns 

994 ------- 

995 names : `~collections.abc.KeysView` 

996 List of `lsst.pex.config.Field` names. 

997 """ 

998 return self._storage.keys() 

999 

1000 def values(self): 

1001 """Get field values. 

1002 

1003 Returns 

1004 ------- 

1005 values : `~collections.abc.ValuesView` 

1006 Iterator of field values. 

1007 """ 

1008 return self._storage.values() 

1009 

1010 def items(self): 

1011 """Get configurations as ``(field name, field value)`` pairs. 

1012 

1013 Returns 

1014 ------- 

1015 items : `~collections.abc.ItemsView` 

1016 Iterator of tuples for each configuration. Tuple items are: 

1017 

1018 0. Field name. 

1019 1. Field value. 

1020 """ 

1021 return self._storage.items() 

1022 

1023 def __contains__(self, name): 

1024 """Return `True` if the specified field exists in this config. 

1025 

1026 Parameters 

1027 ---------- 

1028 name : `str` 

1029 Field name to test for. 

1030 

1031 Returns 

1032 ------- 

1033 in : `bool` 

1034 `True` if the specified field exists in the config. 

1035 """ 

1036 return self._storage.__contains__(name) 

1037 

1038 def __new__(cls, *args, **kw): 

1039 """Allocate a new `lsst.pex.config.Config` object. 

1040 

1041 In order to ensure that all Config object are always in a proper state 

1042 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1043 some attributes are handled at allocation time rather than at 

1044 initialization. 

1045 

1046 This ensures that even if a derived `~lsst.pex.config.Config` class 

1047 implements ``__init__``, its author does not need to be concerned about 

1048 when or even the base ``Config.__init__`` should be called. 

1049 """ 

1050 name = kw.pop("__name", None) 

1051 at = kw.pop("__at", getCallStack()) 

1052 # remove __label and ignore it 

1053 kw.pop("__label", "default") 

1054 

1055 instance = object.__new__(cls) 

1056 instance._frozen = False 

1057 instance._name = name 

1058 instance._storage = {} 

1059 instance._history = {} 

1060 instance._imports = set() 

1061 # load up defaults 

1062 for field in instance._fields.values(): 

1063 instance._history[field.name] = [] 

1064 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1065 # set custom default-overrides 

1066 instance.setDefaults() 

1067 # set constructor overrides 

1068 instance.update(__at=at, **kw) 

1069 return instance 

1070 

1071 def __reduce__(self): 

1072 """Reduction for pickling (function with arguments to reproduce). 

1073 

1074 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1075 since it may contain lambdas (as the ``check`` elements) that cannot 

1076 be pickled. 

1077 """ 

1078 # The stream must be in characters to match the API but pickle 

1079 # requires bytes 

1080 stream = io.StringIO() 

1081 self.saveToStream(stream) 

1082 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1083 

1084 def setDefaults(self): 

1085 """Subclass hook for computing defaults. 

1086 

1087 Notes 

1088 ----- 

1089 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1090 rather than using the `~lsst.pex.config.Field` instances's defaults 

1091 should do so here. To correctly use inherited defaults, 

1092 implementations of ``setDefaults`` must call their base class's 

1093 ``setDefaults``. 

1094 """ 

1095 pass 

1096 

1097 def update(self, **kw): 

1098 """Update values of fields specified by the keyword arguments. 

1099 

1100 Parameters 

1101 ---------- 

1102 **kw 

1103 Keywords are configuration field names. Values are configuration 

1104 field values. 

1105 

1106 Notes 

1107 ----- 

1108 The ``__at`` and ``__label`` keyword arguments are special internal 

1109 keywords. They are used to strip out any internal steps from the 

1110 history tracebacks of the config. Do not modify these keywords to 

1111 subvert a `~lsst.pex.config.Config` instance's history. 

1112 

1113 Examples 

1114 -------- 

1115 This is a config with three fields: 

1116 

1117 >>> from lsst.pex.config import Config, Field 

1118 >>> class DemoConfig(Config): 

1119 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1120 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1121 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1122 ... 

1123 >>> config = DemoConfig() 

1124 

1125 These are the default values of each field: 

1126 

1127 >>> for name, value in config.iteritems(): 

1128 ... print(f"{name}: {value}") 

1129 ... 

1130 fieldA: 42 

1131 fieldB: True 

1132 fieldC: 'Hello world' 

1133 

1134 Using this method to update ``fieldA`` and ``fieldC``: 

1135 

1136 >>> config.update(fieldA=13, fieldC='Updated!') 

1137 

1138 Now the values of each field are: 

1139 

1140 >>> for name, value in config.iteritems(): 

1141 ... print(f"{name}: {value}") 

1142 ... 

1143 fieldA: 13 

1144 fieldB: True 

1145 fieldC: 'Updated!' 

1146 """ 

1147 at = kw.pop("__at", getCallStack()) 

1148 label = kw.pop("__label", "update") 

1149 

1150 for name, value in kw.items(): 

1151 try: 

1152 field = self._fields[name] 

1153 field.__set__(self, value, at=at, label=label) 

1154 except KeyError: 

1155 raise KeyError(f"No field of name {name} exists in config type {_typeStr(self)}") 

1156 

1157 def load(self, filename, root="config"): 

1158 """Modify this config in place by executing the Python code in a 

1159 configuration file. 

1160 

1161 Parameters 

1162 ---------- 

1163 filename : `str` 

1164 Name of the configuration file. A configuration file is Python 

1165 module. 

1166 root : `str`, optional 

1167 Name of the variable in file that refers to the config being 

1168 overridden. 

1169 

1170 For example, the value of root is ``"config"`` and the file 

1171 contains:: 

1172 

1173 config.myField = 5 

1174 

1175 Then this config's field ``myField`` is set to ``5``. 

1176 

1177 See Also 

1178 -------- 

1179 lsst.pex.config.Config.loadFromStream 

1180 lsst.pex.config.Config.loadFromString 

1181 lsst.pex.config.Config.save 

1182 lsst.pex.config.Config.saveToStream 

1183 lsst.pex.config.Config.saveToString 

1184 """ 

1185 with open(filename) as f: 

1186 code = compile(f.read(), filename=filename, mode="exec") 

1187 self.loadFromString(code, root=root, filename=filename) 

1188 

1189 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None): 

1190 """Modify this Config in place by executing the Python code in the 

1191 provided stream. 

1192 

1193 Parameters 

1194 ---------- 

1195 stream : file-like object, `str`, `bytes`, or `~types.CodeType` 

1196 Stream containing configuration override code. If this is a 

1197 code object, it should be compiled with ``mode="exec"``. 

1198 root : `str`, optional 

1199 Name of the variable in file that refers to the config being 

1200 overridden. 

1201 

1202 For example, the value of root is ``"config"`` and the file 

1203 contains:: 

1204 

1205 config.myField = 5 

1206 

1207 Then this config's field ``myField`` is set to ``5``. 

1208 filename : `str`, optional 

1209 Name of the configuration file, or `None` if unknown or contained 

1210 in the stream. Used for error reporting. 

1211 extraLocals : `dict` of `str` to `object`, optional 

1212 Any extra variables to include in local scope when loading. 

1213 

1214 Notes 

1215 ----- 

1216 For backwards compatibility reasons, this method accepts strings, bytes 

1217 and code objects as well as file-like objects. New code should use 

1218 `loadFromString` instead for most of these types. 

1219 

1220 See Also 

1221 -------- 

1222 lsst.pex.config.Config.load 

1223 lsst.pex.config.Config.loadFromString 

1224 lsst.pex.config.Config.save 

1225 lsst.pex.config.Config.saveToStream 

1226 lsst.pex.config.Config.saveToString 

1227 """ 

1228 if hasattr(stream, "read"): 1228 ↛ 1229line 1228 didn't jump to line 1229, because the condition on line 1228 was never true

1229 if filename is None: 

1230 filename = getattr(stream, "name", "?") 

1231 code = compile(stream.read(), filename=filename, mode="exec") 

1232 else: 

1233 code = stream 

1234 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals) 

1235 

1236 def loadFromString(self, code, root="config", filename=None, extraLocals=None): 

1237 """Modify this Config in place by executing the Python code in the 

1238 provided string. 

1239 

1240 Parameters 

1241 ---------- 

1242 code : `str`, `bytes`, or `~types.CodeType` 

1243 Stream containing configuration override code. 

1244 root : `str`, optional 

1245 Name of the variable in file that refers to the config being 

1246 overridden. 

1247 

1248 For example, the value of root is ``"config"`` and the file 

1249 contains:: 

1250 

1251 config.myField = 5 

1252 

1253 Then this config's field ``myField`` is set to ``5``. 

1254 filename : `str`, optional 

1255 Name of the configuration file, or `None` if unknown or contained 

1256 in the stream. Used for error reporting. 

1257 extraLocals : `dict` of `str` to `object`, optional 

1258 Any extra variables to include in local scope when loading. 

1259 

1260 Raises 

1261 ------ 

1262 ValueError 

1263 Raised if a key in extraLocals is the same value as the value of 

1264 the root argument. 

1265 

1266 See Also 

1267 -------- 

1268 lsst.pex.config.Config.load 

1269 lsst.pex.config.Config.loadFromStream 

1270 lsst.pex.config.Config.save 

1271 lsst.pex.config.Config.saveToStream 

1272 lsst.pex.config.Config.saveToString 

1273 """ 

1274 if filename is None: 1274 ↛ 1278line 1274 didn't jump to line 1278, because the condition on line 1274 was never false

1275 # try to determine the file name; a compiled string 

1276 # has attribute "co_filename", 

1277 filename = getattr(code, "co_filename", "?") 

1278 with RecordingImporter() as importer: 

1279 globals = {"__file__": filename} 

1280 local = {root: self} 

1281 if extraLocals is not None: 1281 ↛ 1283line 1281 didn't jump to line 1283, because the condition on line 1281 was never true

1282 # verify the value of root was not passed as extra local args 

1283 if root in extraLocals: 

1284 raise ValueError( 

1285 f"{root} is reserved and cannot be used as a variable name in extraLocals" 

1286 ) 

1287 local.update(extraLocals) 

1288 exec(code, globals, local) 

1289 

1290 self._imports.update(importer.getModules()) 

1291 

1292 def save(self, filename, root="config"): 

1293 """Save a Python script to the named file, which, when loaded, 

1294 reproduces this config. 

1295 

1296 Parameters 

1297 ---------- 

1298 filename : `str` 

1299 Desination filename of this configuration. 

1300 root : `str`, optional 

1301 Name to use for the root config variable. The same value must be 

1302 used when loading (see `lsst.pex.config.Config.load`). 

1303 

1304 See Also 

1305 -------- 

1306 lsst.pex.config.Config.saveToStream 

1307 lsst.pex.config.Config.saveToString 

1308 lsst.pex.config.Config.load 

1309 lsst.pex.config.Config.loadFromStream 

1310 lsst.pex.config.Config.loadFromString 

1311 """ 

1312 d = os.path.dirname(filename) 

1313 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1314 self.saveToStream(outfile, root) 

1315 # tempfile is hardcoded to create files with mode '0600' 

1316 # for an explantion of these antics see: 

1317 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1318 umask = os.umask(0o077) 

1319 os.umask(umask) 

1320 os.chmod(outfile.name, (~umask & 0o666)) 

1321 # chmod before the move so we get quasi-atomic behavior if the 

1322 # source and dest. are on the same filesystem. 

1323 # os.rename may not work across filesystems 

1324 shutil.move(outfile.name, filename) 

1325 

1326 def saveToString(self, skipImports=False): 

1327 """Return the Python script form of this configuration as an executable 

1328 string. 

1329 

1330 Parameters 

1331 ---------- 

1332 skipImports : `bool`, optional 

1333 If `True` then do not include ``import`` statements in output, 

1334 this is to support human-oriented output from ``pipetask`` where 

1335 additional clutter is not useful. 

1336 

1337 Returns 

1338 ------- 

1339 code : `str` 

1340 A code string readable by `loadFromString`. 

1341 

1342 See Also 

1343 -------- 

1344 lsst.pex.config.Config.save 

1345 lsst.pex.config.Config.saveToStream 

1346 lsst.pex.config.Config.load 

1347 lsst.pex.config.Config.loadFromStream 

1348 lsst.pex.config.Config.loadFromString 

1349 """ 

1350 buffer = io.StringIO() 

1351 self.saveToStream(buffer, skipImports=skipImports) 

1352 return buffer.getvalue() 

1353 

1354 def saveToStream(self, outfile, root="config", skipImports=False): 

1355 """Save a configuration file to a stream, which, when loaded, 

1356 reproduces this config. 

1357 

1358 Parameters 

1359 ---------- 

1360 outfile : file-like object 

1361 Destination file object write the config into. Accepts strings not 

1362 bytes. 

1363 root : `str`, optional 

1364 Name to use for the root config variable. The same value must be 

1365 used when loading (see `lsst.pex.config.Config.load`). 

1366 skipImports : `bool`, optional 

1367 If `True` then do not include ``import`` statements in output, 

1368 this is to support human-oriented output from ``pipetask`` where 

1369 additional clutter is not useful. 

1370 

1371 See Also 

1372 -------- 

1373 lsst.pex.config.Config.save 

1374 lsst.pex.config.Config.saveToString 

1375 lsst.pex.config.Config.load 

1376 lsst.pex.config.Config.loadFromStream 

1377 lsst.pex.config.Config.loadFromString 

1378 """ 

1379 tmp = self._name 

1380 self._rename(root) 

1381 try: 

1382 if not skipImports: 1382 ↛ 1396line 1382 didn't jump to line 1396, because the condition on line 1382 was never false

1383 self._collectImports() 

1384 # Remove self from the set, as it is handled explicitly below 

1385 self._imports.remove(self.__module__) 

1386 configType = type(self) 

1387 typeString = _typeStr(configType) 

1388 outfile.write(f"import {configType.__module__}\n") 

1389 outfile.write( 

1390 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1391 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1392 ) 

1393 for imp in sorted(self._imports): 1393 ↛ 1394line 1393 didn't jump to line 1394, because the loop on line 1393 never started

1394 if imp in sys.modules and sys.modules[imp] is not None: 

1395 outfile.write(f"import {imp}\n") 

1396 self._save(outfile) 

1397 finally: 

1398 self._rename(tmp) 

1399 

1400 def freeze(self): 

1401 """Make this config, and all subconfigs, read-only.""" 

1402 self._frozen = True 

1403 for field in self._fields.values(): 

1404 field.freeze(self) 

1405 

1406 def _save(self, outfile): 

1407 """Save this config to an open stream object. 

1408 

1409 Parameters 

1410 ---------- 

1411 outfile : file-like object 

1412 Destination file object write the config into. Accepts strings not 

1413 bytes. 

1414 """ 

1415 for field in self._fields.values(): 

1416 field.save(outfile, self) 

1417 

1418 def _collectImports(self): 

1419 """Add module containing self to the list of things to import and 

1420 then loops over all the fields in the config calling a corresponding 

1421 collect method. 

1422 

1423 The field method will call _collectImports on any 

1424 configs it may own and return the set of things to import. This 

1425 returned set will be merged with the set of imports for this config 

1426 class. 

1427 """ 

1428 self._imports.add(self.__module__) 

1429 for name, field in self._fields.items(): 

1430 field._collectImports(self, self._imports) 

1431 

1432 def toDict(self): 

1433 """Make a dictionary of field names and their values. 

1434 

1435 Returns 

1436 ------- 

1437 dict_ : `dict` 

1438 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1439 Values are `~lsst.pex.config.Field` values. 

1440 

1441 See Also 

1442 -------- 

1443 lsst.pex.config.Field.toDict 

1444 

1445 Notes 

1446 ----- 

1447 This method uses the `~lsst.pex.config.Field.toDict` method of 

1448 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1449 implement a ``toDict`` method for *this* method to work. 

1450 """ 

1451 dict_ = {} 

1452 for name, field in self._fields.items(): 

1453 dict_[name] = field.toDict(self) 

1454 return dict_ 

1455 

1456 def names(self): 

1457 """Get all the field names in the config, recursively. 

1458 

1459 Returns 

1460 ------- 

1461 names : `list` of `str` 

1462 Field names. 

1463 """ 

1464 # 

1465 # Rather than sort out the recursion all over again use the 

1466 # pre-existing saveToStream() 

1467 # 

1468 with io.StringIO() as strFd: 

1469 self.saveToStream(strFd, "config") 

1470 contents = strFd.getvalue() 

1471 strFd.close() 

1472 # 

1473 # Pull the names out of the dumped config 

1474 # 

1475 keys = [] 

1476 for line in contents.split("\n"): 

1477 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1478 continue 

1479 

1480 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1481 if mat: 

1482 keys.append(mat.group(1)) 

1483 

1484 return keys 

1485 

1486 def _rename(self, name): 

1487 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1488 

1489 Parameters 

1490 ---------- 

1491 name : `str` 

1492 New name for this config in its parent `~lsst.pex.config.Config`. 

1493 

1494 Notes 

1495 ----- 

1496 This method uses the `~lsst.pex.config.Field.rename` method of 

1497 individual `lsst.pex.config.Field` instances. 

1498 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1499 method for *this* method to work. 

1500 

1501 See Also 

1502 -------- 

1503 lsst.pex.config.Field.rename 

1504 """ 

1505 self._name = name 

1506 for field in self._fields.values(): 

1507 field.rename(self) 

1508 

1509 def validate(self): 

1510 """Validate the Config, raising an exception if invalid. 

1511 

1512 Raises 

1513 ------ 

1514 lsst.pex.config.FieldValidationError 

1515 Raised if verification fails. 

1516 

1517 Notes 

1518 ----- 

1519 The base class implementation performs type checks on all fields by 

1520 calling their `~lsst.pex.config.Field.validate` methods. 

1521 

1522 Complex single-field validation can be defined by deriving new Field 

1523 types. For convenience, some derived `lsst.pex.config.Field`-types 

1524 (`~lsst.pex.config.ConfigField` and 

1525 `~lsst.pex.config.ConfigChoiceField`) are defined in 

1526 ``lsst.pex.config`` that handle recursing into subconfigs. 

1527 

1528 Inter-field relationships should only be checked in derived 

1529 `~lsst.pex.config.Config` classes after calling this method, and base 

1530 validation is complete. 

1531 """ 

1532 for field in self._fields.values(): 

1533 field.validate(self) 

1534 

1535 def formatHistory(self, name, **kwargs): 

1536 """Format a configuration field's history to a human-readable format. 

1537 

1538 Parameters 

1539 ---------- 

1540 name : `str` 

1541 Name of a `~lsst.pex.config.Field` in this config. 

1542 **kwargs 

1543 Keyword arguments passed to `lsst.pex.config.history.format`. 

1544 

1545 Returns 

1546 ------- 

1547 history : `str` 

1548 A string containing the formatted history. 

1549 

1550 See Also 

1551 -------- 

1552 lsst.pex.config.history.format 

1553 """ 

1554 import lsst.pex.config.history as pexHist 

1555 

1556 return pexHist.format(self, name, **kwargs) 

1557 

1558 history = property(lambda x: x._history) 1558 ↛ exitline 1558 didn't run the lambda on line 1558

1559 """Read-only history. 

1560 """ 

1561 

1562 def __setattr__(self, attr, value, at=None, label="assignment"): 

1563 """Set an attribute (such as a field's value). 

1564 

1565 Notes 

1566 ----- 

1567 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1568 locked such that no additional attributes nor properties may be added 

1569 to them dynamically. 

1570 

1571 Although this is not the standard Python behavior, it helps to protect 

1572 users from accidentally mispelling a field name, or trying to set a 

1573 non-existent field. 

1574 """ 

1575 if attr in self._fields: 

1576 if self._fields[attr].deprecated is not None: 1576 ↛ 1577line 1576 didn't jump to line 1577, because the condition on line 1576 was never true

1577 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1578 warnings.warn( 

1579 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1580 FutureWarning, 

1581 stacklevel=2, 

1582 ) 

1583 if at is None: 1583 ↛ 1586line 1583 didn't jump to line 1586, because the condition on line 1583 was never false

1584 at = getCallStack() 

1585 # This allows Field descriptors to work. 

1586 self._fields[attr].__set__(self, value, at=at, label=label) 

1587 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1587 ↛ 1589line 1587 didn't jump to line 1589, because the condition on line 1587 was never true

1588 # This allows properties and other non-Field descriptors to work. 

1589 return object.__setattr__(self, attr, value) 

1590 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1590 ↛ 1595line 1590 didn't jump to line 1595, because the condition on line 1590 was never false

1591 # This allows specific private attributes to work. 

1592 self.__dict__[attr] = value 

1593 else: 

1594 # We throw everything else. 

1595 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}") 

1596 

1597 def __delattr__(self, attr, at=None, label="deletion"): 

1598 if attr in self._fields: 

1599 if at is None: 

1600 at = getCallStack() 

1601 self._fields[attr].__delete__(self, at=at, label=label) 

1602 else: 

1603 object.__delattr__(self, attr) 

1604 

1605 def __eq__(self, other): 

1606 if type(other) is type(self): 1606 ↛ 1607line 1606 didn't jump to line 1607, because the condition on line 1606 was never true

1607 for name in self._fields: 

1608 thisValue = getattr(self, name) 

1609 otherValue = getattr(other, name) 

1610 if isinstance(thisValue, float) and math.isnan(thisValue): 

1611 if not math.isnan(otherValue): 

1612 return False 

1613 elif thisValue != otherValue: 

1614 return False 

1615 return True 

1616 return False 

1617 

1618 def __ne__(self, other): 

1619 return not self.__eq__(other) 

1620 

1621 def __str__(self): 

1622 return str(self.toDict()) 

1623 

1624 def __repr__(self): 

1625 return "{}({})".format( 

1626 _typeStr(self), 

1627 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None), 

1628 ) 

1629 

1630 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1631 """Compare this configuration to another `~lsst.pex.config.Config` for 

1632 equality. 

1633 

1634 Parameters 

1635 ---------- 

1636 other : `lsst.pex.config.Config` 

1637 Other `~lsst.pex.config.Config` object to compare against this 

1638 config. 

1639 shortcut : `bool`, optional 

1640 If `True`, return as soon as an inequality is found. Default is 

1641 `True`. 

1642 rtol : `float`, optional 

1643 Relative tolerance for floating point comparisons. 

1644 atol : `float`, optional 

1645 Absolute tolerance for floating point comparisons. 

1646 output : callable, optional 

1647 A callable that takes a string, used (possibly repeatedly) to 

1648 report inequalities. 

1649 

1650 Returns 

1651 ------- 

1652 isEqual : `bool` 

1653 `True` when the two `lsst.pex.config.Config` instances are equal. 

1654 `False` if there is an inequality. 

1655 

1656 See Also 

1657 -------- 

1658 lsst.pex.config.compareConfigs 

1659 

1660 Notes 

1661 ----- 

1662 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1663 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1664 are not considered by this method. 

1665 

1666 Floating point comparisons are performed by `numpy.allclose`. 

1667 """ 

1668 name1 = self._name if self._name is not None else "config" 

1669 name2 = other._name if other._name is not None else "config" 

1670 name = getComparisonName(name1, name2) 

1671 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1672 

1673 @classmethod 

1674 def __init_subclass__(cls, **kwargs): 

1675 """Run initialization for every subclass. 

1676 

1677 Specifically registers the subclass with a YAML representer 

1678 and YAML constructor (if pyyaml is available) 

1679 """ 

1680 super().__init_subclass__(**kwargs) 

1681 

1682 if not yaml: 1682 ↛ 1683line 1682 didn't jump to line 1683, because the condition on line 1682 was never true

1683 return 

1684 

1685 yaml.add_representer(cls, _yaml_config_representer) 

1686 

1687 @classmethod 

1688 def _fromPython(cls, config_py): 

1689 """Instantiate a `Config`-subclass from serialized Python form. 

1690 

1691 Parameters 

1692 ---------- 

1693 config_py : `str` 

1694 A serialized form of the Config as created by 

1695 `Config.saveToStream`. 

1696 

1697 Returns 

1698 ------- 

1699 config : `Config` 

1700 Reconstructed `Config` instant. 

1701 """ 

1702 cls = _classFromPython(config_py) 

1703 return unreduceConfig(cls, config_py) 

1704 

1705 

1706def _classFromPython(config_py): 

1707 """Return the Config subclass required by this Config serialization. 

1708 

1709 Parameters 

1710 ---------- 

1711 config_py : `str` 

1712 A serialized form of the Config as created by 

1713 `Config.saveToStream`. 

1714 

1715 Returns 

1716 ------- 

1717 cls : `type` 

1718 The `Config` subclass associated with this config. 

1719 """ 

1720 # standard serialization has the form: 

1721 # import config.class 

1722 # assert type(config)==config.class.Config, ... 

1723 # We want to parse these two lines so we can get the class itself 

1724 

1725 # Do a single regex to avoid large string copies when splitting a 

1726 # large config into separate lines. 

1727 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1728 

1729 if not matches: 

1730 first_line, second_line, _ = config_py.split("\n", 2) 

1731 raise ValueError( 

1732 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}" 

1733 ) 

1734 

1735 module_name = matches.group(1) 

1736 module = importlib.import_module(module_name) 

1737 

1738 # Second line 

1739 full_name = matches.group(2) 

1740 

1741 # Remove the module name from the full name 

1742 if not full_name.startswith(module_name): 

1743 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1744 

1745 # if module name is a.b.c and full name is a.b.c.d.E then 

1746 # we need to remove a.b.c. and iterate over the remainder 

1747 # The +1 is for the extra dot after a.b.c 

1748 remainder = full_name[len(module_name) + 1 :] 

1749 components = remainder.split(".") 

1750 pytype = module 

1751 for component in components: 

1752 pytype = getattr(pytype, component) 

1753 return pytype 

1754 

1755 

1756def unreduceConfig(cls_, stream): 

1757 """Create a `~lsst.pex.config.Config` from a stream. 

1758 

1759 Parameters 

1760 ---------- 

1761 cls_ : `lsst.pex.config.Config`-type 

1762 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1763 with configurations in the ``stream``. 

1764 stream : file-like object, `str`, or `~types.CodeType` 

1765 Stream containing configuration override code. 

1766 

1767 Returns 

1768 ------- 

1769 config : `lsst.pex.config.Config` 

1770 Config instance. 

1771 

1772 See Also 

1773 -------- 

1774 lsst.pex.config.Config.loadFromStream 

1775 """ 

1776 config = cls_() 

1777 config.loadFromStream(stream) 

1778 return config