Coverage for python/lsst/pex/config/config.py: 58%

462 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-16 10:06 +0000

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "Config", 

31 "ConfigMeta", 

32 "Field", 

33 "FieldValidationError", 

34 "UnexpectedProxyUsageError", 

35 "FieldTypeVar", 

36) 

37 

38import copy 

39import importlib 

40import io 

41import math 

42import os 

43import re 

44import shutil 

45import sys 

46import tempfile 

47import warnings 

48from collections.abc import Mapping 

49from typing import Any, ForwardRef, Generic, TypeVar, cast, overload 

50 

51try: 

52 from types import GenericAlias 

53except ImportError: 

54 # cover python 3.8 usage 

55 GenericAlias = type(Mapping[int, int]) 

56 

57# if YAML is not available that's fine and we simply don't register 

58# the yaml representer since we know it won't be used. 

59try: 

60 import yaml 

61except ImportError: 

62 yaml = None 

63 

64from .callStack import getCallStack, getStackFrame 

65from .comparison import compareConfigs, compareScalars, getComparisonName 

66 

67if yaml: 67 ↛ 78line 67 didn't jump to line 78, because the condition on line 67 was never false

68 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader) 

69 

70 try: 

71 # CLoader is not always available 

72 from yaml import CLoader 

73 

74 YamlLoaders += (CLoader,) 

75 except ImportError: 

76 pass 

77else: 

78 YamlLoaders = () 

79 doImport = None 

80 

81 

82class _PexConfigGenericAlias(GenericAlias): 

83 """A Subclass of python's GenericAlias used in defining and instantiating 

84 Generics. 

85 

86 This class differs from `types.GenericAlias` in that it calls a method 

87 named _parseTypingArgs defined on Fields. This method gives Field and its 

88 subclasses an opportunity to transform type parameters into class key word 

89 arguments. Code authors do not need to implement any returns of this object 

90 directly, and instead only need implement _parseTypingArgs, if a Field 

91 subclass differs from the base class implementation. 

92 

93 This class is intended to be an implementation detail, returned from a 

94 Field's `__class_getitem__` method. 

95 """ 

96 

97 def __call__(self, *args: Any, **kwds: Any) -> Any: 

98 origin_kwargs = self._parseTypingArgs(self.__args__, kwds) 

99 return super().__call__(*args, **{**kwds, **origin_kwargs}) 

100 

101 

102FieldTypeVar = TypeVar("FieldTypeVar") 

103 

104 

105class UnexpectedProxyUsageError(TypeError): 

106 """Exception raised when a proxy class is used in a context that suggests 

107 it should have already been converted to the thing it proxies. 

108 """ 

109 

110 

111def _joinNamePath(prefix=None, name=None, index=None): 

112 """Generate nested configuration names.""" 

113 if not prefix and not name: 113 ↛ 114line 113 didn't jump to line 114, because the condition on line 113 was never true

114 raise ValueError("Invalid name: cannot be None") 

115 elif not name: 115 ↛ 116line 115 didn't jump to line 116, because the condition on line 115 was never true

116 name = prefix 

117 elif prefix and name: 117 ↛ 120line 117 didn't jump to line 120, because the condition on line 117 was never false

118 name = prefix + "." + name 

119 

120 if index is not None: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true

121 return f"{name}[{index!r}]" 

122 else: 

123 return name 

124 

125 

126def _autocast(x, dtype): 

127 """Cast a value to a type, if appropriate. 

128 

129 Parameters 

130 ---------- 

131 x : object 

132 A value. 

133 dtype : tpye 

134 Data type, such as `float`, `int`, or `str`. 

135 

136 Returns 

137 ------- 

138 values : object 

139 If appropriate, the returned value is ``x`` cast to the given type 

140 ``dtype``. If the cast cannot be performed the original value of 

141 ``x`` is returned. 

142 """ 

143 if dtype == float and isinstance(x, int): 

144 return float(x) 

145 return x 

146 

147 

148def _typeStr(x): 

149 """Generate a fully-qualified type name. 

150 

151 Returns 

152 ------- 

153 `str` 

154 Fully-qualified type name. 

155 

156 Notes 

157 ----- 

158 This function is used primarily for writing config files to be executed 

159 later upon with the 'load' function. 

160 """ 

161 if hasattr(x, "__module__") and hasattr(x, "__name__"): 

162 xtype = x 

163 else: 

164 xtype = type(x) 

165 if xtype.__module__ == "builtins": 165 ↛ 166line 165 didn't jump to line 166, because the condition on line 165 was never true

166 return xtype.__name__ 

167 else: 

168 return f"{xtype.__module__}.{xtype.__name__}" 

169 

170 

171if yaml: 171 ↛ 204line 171 didn't jump to line 204, because the condition on line 171 was never false

172 

173 def _yaml_config_representer(dumper, data): 

174 """Represent a Config object in a form suitable for YAML. 

175 

176 Stores the serialized stream as a scalar block string. 

177 """ 

178 stream = io.StringIO() 

179 data.saveToStream(stream) 

180 config_py = stream.getvalue() 

181 

182 # Strip multiple newlines from the end of the config 

183 # This simplifies the YAML to use | and not |+ 

184 config_py = config_py.rstrip() + "\n" 

185 

186 # Trailing spaces force pyyaml to use non-block form. 

187 # Remove the trailing spaces so it has no choice 

188 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE) 

189 

190 # Store the Python as a simple scalar 

191 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|") 

192 

193 def _yaml_config_constructor(loader, node): 

194 """Construct a config from YAML.""" 

195 config_py = loader.construct_scalar(node) 

196 return Config._fromPython(config_py) 

197 

198 # Register a generic constructor for Config and all subclasses 

199 # Need to register for all the loaders we would like to use 

200 for loader in YamlLoaders: 

201 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader) 

202 

203 

204class ConfigMeta(type): 

205 """A metaclass for `lsst.pex.config.Config`. 

206 

207 Notes 

208 ----- 

209 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

210 class attributes as a class attribute called ``_fields``, and adds 

211 the name of each field as an instance variable of the field itself (so you 

212 don't have to pass the name of the field to the field constructor). 

213 """ 

214 

215 def __init__(cls, name, bases, dict_): 

216 type.__init__(cls, name, bases, dict_) 

217 cls._fields = {} 

218 cls._source = getStackFrame() 

219 

220 def getFields(classtype): 

221 fields = {} 

222 bases = list(classtype.__bases__) 

223 bases.reverse() 

224 for b in bases: 

225 fields.update(getFields(b)) 

226 

227 for k, v in classtype.__dict__.items(): 

228 if isinstance(v, Field): 

229 fields[k] = v 

230 return fields 

231 

232 fields = getFields(cls) 

233 for k, v in fields.items(): 

234 setattr(cls, k, copy.deepcopy(v)) 

235 

236 def __setattr__(cls, name, value): 

237 if isinstance(value, Field): 

238 value.name = name 

239 cls._fields[name] = value 

240 type.__setattr__(cls, name, value) 

241 

242 

243class FieldValidationError(ValueError): 

244 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

245 particular ``~lsst.pex.config.Config``. 

246 

247 Parameters 

248 ---------- 

249 field : `lsst.pex.config.Field` 

250 The field that was not valid. 

251 config : `lsst.pex.config.Config` 

252 The config containing the invalid field. 

253 msg : `str` 

254 Text describing why the field was not valid. 

255 """ 

256 

257 def __init__(self, field, config, msg): 

258 self.fieldType = type(field) 

259 """Type of the `~lsst.pex.config.Field` that incurred the error. 

260 """ 

261 

262 self.fieldName = field.name 

263 """Name of the `~lsst.pex.config.Field` instance that incurred the 

264 error (`str`). 

265 

266 See also 

267 -------- 

268 lsst.pex.config.Field.name 

269 """ 

270 

271 self.fullname = _joinNamePath(config._name, field.name) 

272 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

273 (`str`). 

274 """ 

275 

276 self.history = config.history.setdefault(field.name, []) 

277 """Full history of all changes to the `~lsst.pex.config.Field` 

278 instance. 

279 """ 

280 

281 self.fieldSource = field.source 

282 """File and line number of the `~lsst.pex.config.Field` definition. 

283 """ 

284 

285 self.configSource = config._source 

286 error = ( 

287 "%s '%s' failed validation: %s\n" 

288 "For more information see the Field definition at:\n%s" 

289 " and the Config definition at:\n%s" 

290 % ( 

291 self.fieldType.__name__, 

292 self.fullname, 

293 msg, 

294 self.fieldSource.format(), 

295 self.configSource.format(), 

296 ) 

297 ) 

298 super().__init__(error) 

299 

300 

301class Field(Generic[FieldTypeVar]): 

302 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

303 `complex`, `bool`, and `str` data types. 

304 

305 Parameters 

306 ---------- 

307 doc : `str` 

308 A description of the field for users. 

309 dtype : type, optional 

310 The field's data type. ``Field`` only supports basic data types: 

311 `int`, `float`, `complex`, `bool`, and `str`. See 

312 `Field.supportedTypes`. Optional if supplied as a typing argument to 

313 the class. 

314 default : object, optional 

315 The field's default value. 

316 check : callable, optional 

317 A callable that is called with the field's value. This callable should 

318 return `False` if the value is invalid. More complex inter-field 

319 validation can be written as part of the 

320 `lsst.pex.config.Config.validate` method. 

321 optional : `bool`, optional 

322 This sets whether the field is considered optional, and therefore 

323 doesn't need to be set by the user. When `False`, 

324 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

325 deprecated : None or `str`, optional 

326 A description of why this Field is deprecated, including removal date. 

327 If not None, the string is appended to the docstring for this Field. 

328 

329 Raises 

330 ------ 

331 ValueError 

332 Raised when the ``dtype`` parameter is not one of the supported types 

333 (see `Field.supportedTypes`). 

334 

335 See Also 

336 -------- 

337 ChoiceField 

338 ConfigChoiceField 

339 ConfigDictField 

340 ConfigField 

341 ConfigurableField 

342 DictField 

343 ListField 

344 RangeField 

345 RegistryField 

346 

347 Notes 

348 ----- 

349 ``Field`` instances (including those of any subclass of ``Field``) are used 

350 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

351 example, below). ``Field`` attributes work like the `property` attributes 

352 of classes that implement custom setters and getters. `Field` attributes 

353 belong to the class, but operate on the instance. Formally speaking, 

354 `Field` attributes are `descriptors 

355 <https://docs.python.org/3/howto/descriptor.html>`_. 

356 

357 When you access a `Field` attribute on a `Config` instance, you don't 

358 get the `Field` instance itself. Instead, you get the value of that field, 

359 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

360 container type (like a `lsst.pex.config.List`) depending on the field's 

361 type. See the example, below. 

362 

363 Fields can be annotated with a type similar to other python classes (python 

364 specification `here <https://peps.python.org/pep-0484/#generics>`_ ). 

365 See the name field in the Config example below for an example of this. 

366 Unlike most other uses in python, this has an effect at type checking *and* 

367 runtime. If the type is specified with a class annotation, it will be used 

368 as the value of the ``dtype`` in the ``Field`` and there is no need to 

369 specify it as an argument during instantiation. 

370 

371 There are Some notes on dtype through type annotation syntax. Type 

372 annotation syntax supports supplying the argument as a string of a type 

373 name. i.e. "float", but this cannot be used to resolve circular references. 

374 Type annotation syntax can be used on an identifier in addition to Class 

375 assignment i.e. ``variable: Field[str] = Config.someField`` vs 

376 ``someField = Field[str](doc="some doc"). However, this syntax is only 

377 useful for annotating the type of the identifier (i.e. variable in previous 

378 example) and does nothing for assigning the dtype of the ``Field``. 

379 

380 

381 Examples 

382 -------- 

383 Instances of ``Field`` should be used as class attributes of 

384 `lsst.pex.config.Config` subclasses: 

385 

386 >>> from lsst.pex.config import Config, Field 

387 >>> class Example(Config): 

388 ... myInt = Field("An integer field.", int, default=0) 

389 ... name = Field[str](doc="A string Field") 

390 ... 

391 >>> print(config.myInt) 

392 0 

393 >>> config.myInt = 5 

394 >>> print(config.myInt) 

395 5 

396 """ 

397 

398 name: str 

399 """Identifier (variable name) used to refer to a Field within a Config 

400 Class. 

401 """ 

402 

403 supportedTypes = {str, bool, float, int, complex} 

404 """Supported data types for field values (`set` of types). 

405 """ 

406 

407 @staticmethod 

408 def _parseTypingArgs( 

409 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any] 

410 ) -> Mapping[str, Any]: 

411 """Parse type annotations into keyword constructor arguments. 

412 

413 This is a special private method that interprets type arguments (i.e. 

414 Field[str]) into keyword arguments to be passed on to the constructor. 

415 

416 Subclasses of Field can implement this method to customize how they 

417 handle turning type parameters into keyword arguments (see DictField 

418 for an example) 

419 

420 Parameters 

421 ---------- 

422 params : `tuple` of `type` or `tuple` of str 

423 Parameters passed to the type annotation. These will either be 

424 types or strings. Strings are to interpreted as forward references 

425 and will be treated as such. 

426 kwds : `MutableMapping` with keys of `str` and values of `Any` 

427 These are the user supplied keywords that are to be passed to the 

428 Field constructor. 

429 

430 Returns 

431 ------- 

432 kwds : `MutableMapping` with keys of `str` and values of `Any` 

433 The mapping of keywords that will be passed onto the constructor 

434 of the Field. Should be filled in with any information gleaned 

435 from the input parameters. 

436 

437 Raises 

438 ------ 

439 ValueError : 

440 Raised if params is of incorrect length. 

441 Raised if a forward reference could not be resolved 

442 Raised if there is a conflict between params and values in kwds 

443 """ 

444 if len(params) > 1: 

445 raise ValueError("Only single type parameters are supported") 

446 unpackedParams = params[0] 

447 if isinstance(unpackedParams, str): 

448 _typ = ForwardRef(unpackedParams) 

449 # type ignore below because typeshed seems to be wrong. It 

450 # indicates there are only 2 args, as it was in python 3.8, but 

451 # 3.9+ takes 3 args. Attempt in old style and new style to 

452 # work with both. 

453 try: 

454 result = _typ._evaluate(globals(), locals(), set()) # type: ignore 

455 except TypeError: 

456 # python 3.8 path 

457 result = _typ._evaluate(globals(), locals()) 

458 if result is None: 

459 raise ValueError("Could not deduce type from input") 

460 unpackedParams = cast(type, result) 

461 if "dtype" in kwds and kwds["dtype"] != unpackedParams: 

462 raise ValueError("Conflicting definition for dtype") 

463 elif "dtype" not in kwds: 

464 kwds = {**kwds, **{"dtype": unpackedParams}} 

465 return kwds 

466 

467 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef): 

468 return _PexConfigGenericAlias(cls, params) 

469 

470 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None): 

471 if dtype is None: 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 raise ValueError( 

473 "dtype must either be supplied as an argument or as a type argument to the class" 

474 ) 

475 if dtype not in self.supportedTypes: 475 ↛ 476line 475 didn't jump to line 476, because the condition on line 475 was never true

476 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

477 

478 source = getStackFrame() 

479 self._setup( 

480 doc=doc, 

481 dtype=dtype, 

482 default=default, 

483 check=check, 

484 optional=optional, 

485 source=source, 

486 deprecated=deprecated, 

487 ) 

488 

489 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

490 """Set attributes, usually during initialization.""" 

491 self.dtype = dtype 

492 """Data type for the field. 

493 """ 

494 

495 if not doc: 495 ↛ 496line 495 didn't jump to line 496, because the condition on line 495 was never true

496 raise ValueError("Docstring is empty.") 

497 

498 # append the deprecation message to the docstring. 

499 if deprecated is not None: 

500 doc = f"{doc} Deprecated: {deprecated}" 

501 self.doc = doc 

502 """A description of the field (`str`). 

503 """ 

504 

505 self.deprecated = deprecated 

506 """If not None, a description of why this field is deprecated (`str`). 

507 """ 

508 

509 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

510 if optional or default is not None: 

511 self.__doc__ += f", default ``{default!r}``" 

512 self.__doc__ += ")" 

513 

514 self.default = default 

515 """Default value for this field. 

516 """ 

517 

518 self.check = check 

519 """A user-defined function that validates the value of the field. 

520 """ 

521 

522 self.optional = optional 

523 """Flag that determines if the field is required to be set (`bool`). 

524 

525 When `False`, `lsst.pex.config.Config.validate` will fail if the 

526 field's value is `None`. 

527 """ 

528 

529 self.source = source 

530 """The stack frame where this field is defined (`list` of 

531 `lsst.pex.config.callStack.StackFrame`). 

532 """ 

533 

534 def rename(self, instance): 

535 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use 

536 only). 

537 

538 Parameters 

539 ---------- 

540 instance : `lsst.pex.config.Config` 

541 The config instance that contains this field. 

542 

543 Notes 

544 ----- 

545 This method is invoked by the `lsst.pex.config.Config` object that 

546 contains this field and should not be called directly. 

547 

548 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

549 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should 

550 rename each subconfig with the full field name as generated by 

551 `lsst.pex.config.config._joinNamePath`. 

552 """ 

553 pass 

554 

555 def validate(self, instance): 

556 """Validate the field (for internal use only). 

557 

558 Parameters 

559 ---------- 

560 instance : `lsst.pex.config.Config` 

561 The config instance that contains this field. 

562 

563 Raises 

564 ------ 

565 lsst.pex.config.FieldValidationError 

566 Raised if verification fails. 

567 

568 Notes 

569 ----- 

570 This method provides basic validation: 

571 

572 - Ensures that the value is not `None` if the field is not optional. 

573 - Ensures type correctness. 

574 - Ensures that the user-provided ``check`` function is valid. 

575 

576 Most `~lsst.pex.config.Field` subclasses should call 

577 `lsst.pex.config.Field.validate` if they re-implement 

578 `~lsst.pex.config.Field.validate`. 

579 """ 

580 value = self.__get__(instance) 

581 if not self.optional and value is None: 

582 raise FieldValidationError(self, instance, "Required value cannot be None") 

583 

584 def freeze(self, instance): 

585 """Make this field read-only (for internal use only). 

586 

587 Parameters 

588 ---------- 

589 instance : `lsst.pex.config.Config` 

590 The config instance that contains this field. 

591 

592 Notes 

593 ----- 

594 Freezing is only relevant for fields that hold subconfigs. Fields which 

595 hold subconfigs should freeze each subconfig. 

596 

597 **Subclasses should implement this method.** 

598 """ 

599 pass 

600 

601 def _validateValue(self, value): 

602 """Validate a value. 

603 

604 Parameters 

605 ---------- 

606 value : object 

607 The value being validated. 

608 

609 Raises 

610 ------ 

611 TypeError 

612 Raised if the value's type is incompatible with the field's 

613 ``dtype``. 

614 ValueError 

615 Raised if the value is rejected by the ``check`` method. 

616 """ 

617 if value is None: 617 ↛ 618line 617 didn't jump to line 618, because the condition on line 617 was never true

618 return 

619 

620 if not isinstance(value, self.dtype): 620 ↛ 621line 620 didn't jump to line 621, because the condition on line 620 was never true

621 msg = "Value {} is of incorrect type {}. Expected type {}".format( 

622 value, 

623 _typeStr(value), 

624 _typeStr(self.dtype), 

625 ) 

626 raise TypeError(msg) 

627 if self.check is not None and not self.check(value): 627 ↛ 628line 627 didn't jump to line 628, because the condition on line 627 was never true

628 msg = "Value %s is not a valid value" % str(value) 

629 raise ValueError(msg) 

630 

631 def _collectImports(self, instance, imports): 

632 """Call the _collectImports method on all config 

633 objects the field may own, and union them with the supplied imports 

634 set. 

635 

636 Parameters 

637 ---------- 

638 instance : instance or subclass of `lsst.pex.config.Config` 

639 A config object that has this field defined on it 

640 imports : `set` 

641 Set of python modules that need imported after persistence 

642 """ 

643 pass 

644 

645 def save(self, outfile, instance): 

646 """Save this field to a file (for internal use only). 

647 

648 Parameters 

649 ---------- 

650 outfile : file-like object 

651 A writeable field handle. 

652 instance : `~lsst.pex.config.Config` 

653 The `~lsst.pex.config.Config` instance that contains this field. 

654 

655 Notes 

656 ----- 

657 This method is invoked by the `~lsst.pex.config.Config` object that 

658 contains this field and should not be called directly. 

659 

660 The output consists of the documentation string 

661 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

662 line is formatted as an assignment: ``{fullname}={value}``. 

663 

664 This output can be executed with Python. 

665 """ 

666 value = self.__get__(instance) 

667 fullname = _joinNamePath(instance._name, self.name) 

668 

669 if self.deprecated and value == self.default: 669 ↛ 670line 669 didn't jump to line 670, because the condition on line 669 was never true

670 return 

671 

672 # write full documentation string as comment lines 

673 # (i.e. first character is #) 

674 doc = "# " + str(self.doc).replace("\n", "\n# ") 

675 if isinstance(value, float) and not math.isfinite(value): 675 ↛ 677line 675 didn't jump to line 677, because the condition on line 675 was never true

676 # non-finite numbers need special care 

677 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n") 

678 else: 

679 outfile.write(f"{doc}\n{fullname}={value!r}\n\n") 

680 

681 def toDict(self, instance): 

682 """Convert the field value so that it can be set as the value of an 

683 item in a `dict` (for internal use only). 

684 

685 Parameters 

686 ---------- 

687 instance : `~lsst.pex.config.Config` 

688 The `~lsst.pex.config.Config` that contains this field. 

689 

690 Returns 

691 ------- 

692 value : object 

693 The field's value. See *Notes*. 

694 

695 Notes 

696 ----- 

697 This method invoked by the owning `~lsst.pex.config.Config` object and 

698 should not be called directly. 

699 

700 Simple values are passed through. Complex data structures must be 

701 manipulated. For example, a `~lsst.pex.config.Field` holding a 

702 subconfig should, instead of the subconfig object, return a `dict` 

703 where the keys are the field names in the subconfig, and the values are 

704 the field values in the subconfig. 

705 """ 

706 return self.__get__(instance) 

707 

708 @overload 

709 def __get__( 

710 self, instance: None, owner: Any = None, at: Any = None, label: str = "default" 

711 ) -> Field[FieldTypeVar]: 

712 ... 

713 

714 @overload 

715 def __get__( 

716 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default" 

717 ) -> FieldTypeVar: 

718 ... 

719 

720 def __get__(self, instance, owner=None, at=None, label="default"): 

721 """Define how attribute access should occur on the Config instance 

722 This is invoked by the owning config object and should not be called 

723 directly. 

724 

725 When the field attribute is accessed on a Config class object, it 

726 returns the field object itself in order to allow inspection of 

727 Config classes. 

728 

729 When the field attribute is access on a config instance, the actual 

730 value described by the field (and held by the Config instance) is 

731 returned. 

732 """ 

733 if instance is None: 733 ↛ 734line 733 didn't jump to line 734, because the condition on line 733 was never true

734 return self 

735 else: 

736 # try statements are almost free in python if they succeed 

737 try: 

738 return instance._storage[self.name] 

739 except AttributeError: 

740 if not isinstance(instance, Config): 

741 return self 

742 else: 

743 raise AttributeError( 

744 f"Config {instance} is missing _storage attribute, likely incorrectly initialized" 

745 ) 

746 

747 def __set__( 

748 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment" 

749 ) -> None: 

750 """Set an attribute on the config instance. 

751 

752 Parameters 

753 ---------- 

754 instance : `lsst.pex.config.Config` 

755 The config instance that contains this field. 

756 value : obj 

757 Value to set on this field. 

758 at : `list` of `lsst.pex.config.callStack.StackFrame` 

759 The call stack (created by 

760 `lsst.pex.config.callStack.getCallStack`). 

761 label : `str`, optional 

762 Event label for the history. 

763 

764 Notes 

765 ----- 

766 This method is invoked by the owning `lsst.pex.config.Config` object 

767 and should not be called directly. 

768 

769 Derived `~lsst.pex.config.Field` classes may need to override the 

770 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

771 should follow the following rules: 

772 

773 - Do not allow modification of frozen configs. 

774 - Validate the new value **before** modifying the field. Except if the 

775 new value is `None`. `None` is special and no attempt should be made 

776 to validate it until `lsst.pex.config.Config.validate` is called. 

777 - Do not modify the `~lsst.pex.config.Config` instance to contain 

778 invalid values. 

779 - If the field is modified, update the history of the 

780 `lsst.pex.config.field.Field` to reflect the changes. 

781 

782 In order to decrease the need to implement this method in derived 

783 `~lsst.pex.config.Field` types, value validation is performed in the 

784 `lsst.pex.config.Field._validateValue`. If only the validation step 

785 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

786 implement `lsst.pex.config.Field._validateValue` than to reimplement 

787 ``__set__``. More complicated behavior, however, may require 

788 reimplementation. 

789 """ 

790 if instance._frozen: 790 ↛ 791line 790 didn't jump to line 791, because the condition on line 790 was never true

791 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

792 

793 history = instance._history.setdefault(self.name, []) 

794 if value is not None: 794 ↛ 801line 794 didn't jump to line 801, because the condition on line 794 was never false

795 value = _autocast(value, self.dtype) 

796 try: 

797 self._validateValue(value) 

798 except BaseException as e: 

799 raise FieldValidationError(self, instance, str(e)) 

800 

801 instance._storage[self.name] = value 

802 if at is None: 802 ↛ 803line 802 didn't jump to line 803, because the condition on line 802 was never true

803 at = getCallStack() 

804 history.append((value, at, label)) 

805 

806 def __delete__(self, instance, at=None, label="deletion"): 

807 """Delete an attribute from a `lsst.pex.config.Config` instance. 

808 

809 Parameters 

810 ---------- 

811 instance : `lsst.pex.config.Config` 

812 The config instance that contains this field. 

813 at : `list` of `lsst.pex.config.callStack.StackFrame` 

814 The call stack (created by 

815 `lsst.pex.config.callStack.getCallStack`). 

816 label : `str`, optional 

817 Event label for the history. 

818 

819 Notes 

820 ----- 

821 This is invoked by the owning `~lsst.pex.config.Config` object and 

822 should not be called directly. 

823 """ 

824 if at is None: 

825 at = getCallStack() 

826 self.__set__(instance, None, at=at, label=label) 

827 

828 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

829 """Compare a field (named `Field.name`) in two 

830 `~lsst.pex.config.Config` instances for equality. 

831 

832 Parameters 

833 ---------- 

834 instance1 : `lsst.pex.config.Config` 

835 Left-hand side `Config` instance to compare. 

836 instance2 : `lsst.pex.config.Config` 

837 Right-hand side `Config` instance to compare. 

838 shortcut : `bool`, optional 

839 **Unused.** 

840 rtol : `float`, optional 

841 Relative tolerance for floating point comparisons. 

842 atol : `float`, optional 

843 Absolute tolerance for floating point comparisons. 

844 output : callable, optional 

845 A callable that takes a string, used (possibly repeatedly) to 

846 report inequalities. 

847 

848 Notes 

849 ----- 

850 This method must be overridden by more complex `Field` subclasses. 

851 

852 See Also 

853 -------- 

854 lsst.pex.config.compareScalars 

855 """ 

856 v1 = getattr(instance1, self.name) 

857 v2 = getattr(instance2, self.name) 

858 name = getComparisonName( 

859 _joinNamePath(instance1._name, self.name), _joinNamePath(instance2._name, self.name) 

860 ) 

861 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

862 

863 

864class RecordingImporter: 

865 """Importer (for `sys.meta_path`) that records which modules are being 

866 imported. 

867 

868 *This class does not do any importing itself.* 

869 

870 Examples 

871 -------- 

872 Use this class as a context manager to ensure it is properly uninstalled 

873 when done: 

874 

875 >>> with RecordingImporter() as importer: 

876 ... # import stuff 

877 ... import numpy as np 

878 ... print("Imported: " + importer.getModules()) 

879 """ 

880 

881 def __init__(self): 

882 self._modules = set() 

883 

884 def __enter__(self): 

885 self.origMetaPath = sys.meta_path 

886 sys.meta_path = [self] + sys.meta_path # type: ignore 

887 return self 

888 

889 def __exit__(self, *args): 

890 self.uninstall() 

891 return False # Don't suppress exceptions 

892 

893 def uninstall(self): 

894 """Uninstall the importer.""" 

895 sys.meta_path = self.origMetaPath 

896 

897 def find_spec(self, fullname, path, target=None): 

898 """Find a module. 

899 

900 Called as part of the ``import`` chain of events. 

901 """ 

902 self._modules.add(fullname) 

903 # Return None because we don't do any importing. 

904 return None 

905 

906 def getModules(self): 

907 """Get the set of modules that were imported. 

908 

909 Returns 

910 ------- 

911 modules : `set` of `str` 

912 Set of imported module names. 

913 """ 

914 return self._modules 

915 

916 

917# type ignore because type checker thinks ConfigMeta is Generic when it is not 

918class Config(metaclass=ConfigMeta): # type: ignore 

919 """Base class for configuration (*config*) objects. 

920 

921 Notes 

922 ----- 

923 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

924 instances as class attributes. These are used to define most of the base 

925 class behavior. 

926 

927 ``Config`` implements a mapping API that provides many `dict`-like methods, 

928 such as `keys`, `values`, and `items`. ``Config`` instances also support 

929 the ``in`` operator to test if a field is in the config. Unlike a `dict`, 

930 ``Config`` classes are not subscriptable. Instead, access individual 

931 fields as attributes of the configuration instance. 

932 

933 Examples 

934 -------- 

935 Config classes are subclasses of ``Config`` that have 

936 `~lsst.pex.config.Field` instances (or instances of 

937 `~lsst.pex.config.Field` subclasses) as class attributes: 

938 

939 >>> from lsst.pex.config import Config, Field, ListField 

940 >>> class DemoConfig(Config): 

941 ... intField = Field(doc="An integer field", dtype=int, default=42) 

942 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

943 ... default=['coffee', 'green tea', 'water']) 

944 ... 

945 >>> config = DemoConfig() 

946 

947 Configs support many `dict`-like APIs: 

948 

949 >>> config.keys() 

950 ['intField', 'listField'] 

951 >>> 'intField' in config 

952 True 

953 

954 Individual fields can be accessed as attributes of the configuration: 

955 

956 >>> config.intField 

957 42 

958 >>> config.listField.append('earl grey tea') 

959 >>> print(config.listField) 

960 ['coffee', 'green tea', 'water', 'earl grey tea'] 

961 """ 

962 

963 _storage: dict[str, Any] 

964 _fields: dict[str, Field] 

965 _history: dict[str, list[Any]] 

966 _imports: set[Any] 

967 

968 def __iter__(self): 

969 """Iterate over fields.""" 

970 return self._fields.__iter__() 

971 

972 def keys(self): 

973 """Get field names. 

974 

975 Returns 

976 ------- 

977 names : `~collections.abc.KeysView` 

978 List of `lsst.pex.config.Field` names. 

979 """ 

980 return self._storage.keys() 

981 

982 def values(self): 

983 """Get field values. 

984 

985 Returns 

986 ------- 

987 values : `~collections.abc.ValuesView` 

988 Iterator of field values. 

989 """ 

990 return self._storage.values() 

991 

992 def items(self): 

993 """Get configurations as ``(field name, field value)`` pairs. 

994 

995 Returns 

996 ------- 

997 items : `~collections.abc.ItemsView` 

998 Iterator of tuples for each configuration. Tuple items are: 

999 

1000 0. Field name. 

1001 1. Field value. 

1002 """ 

1003 return self._storage.items() 

1004 

1005 def __contains__(self, name): 

1006 """Return `True` if the specified field exists in this config. 

1007 

1008 Parameters 

1009 ---------- 

1010 name : `str` 

1011 Field name to test for. 

1012 

1013 Returns 

1014 ------- 

1015 in : `bool` 

1016 `True` if the specified field exists in the config. 

1017 """ 

1018 return self._storage.__contains__(name) 

1019 

1020 def __new__(cls, *args, **kw): 

1021 """Allocate a new `lsst.pex.config.Config` object. 

1022 

1023 In order to ensure that all Config object are always in a proper state 

1024 when handed to users or to derived `~lsst.pex.config.Config` classes, 

1025 some attributes are handled at allocation time rather than at 

1026 initialization. 

1027 

1028 This ensures that even if a derived `~lsst.pex.config.Config` class 

1029 implements ``__init__``, its author does not need to be concerned about 

1030 when or even the base ``Config.__init__`` should be called. 

1031 """ 

1032 name = kw.pop("__name", None) 

1033 at = kw.pop("__at", getCallStack()) 

1034 # remove __label and ignore it 

1035 kw.pop("__label", "default") 

1036 

1037 instance = object.__new__(cls) 

1038 instance._frozen = False 

1039 instance._name = name 

1040 instance._storage = {} 

1041 instance._history = {} 

1042 instance._imports = set() 

1043 # load up defaults 

1044 for field in instance._fields.values(): 

1045 instance._history[field.name] = [] 

1046 field.__set__(instance, field.default, at=at + [field.source], label="default") 

1047 # set custom default-overrides 

1048 instance.setDefaults() 

1049 # set constructor overrides 

1050 instance.update(__at=at, **kw) 

1051 return instance 

1052 

1053 def __reduce__(self): 

1054 """Reduction for pickling (function with arguments to reproduce). 

1055 

1056 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

1057 since it may contain lambdas (as the ``check`` elements) that cannot 

1058 be pickled. 

1059 """ 

1060 # The stream must be in characters to match the API but pickle 

1061 # requires bytes 

1062 stream = io.StringIO() 

1063 self.saveToStream(stream) 

1064 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

1065 

1066 def setDefaults(self): 

1067 """Subclass hook for computing defaults. 

1068 

1069 Notes 

1070 ----- 

1071 Derived `~lsst.pex.config.Config` classes that must compute defaults 

1072 rather than using the `~lsst.pex.config.Field` instances's defaults 

1073 should do so here. To correctly use inherited defaults, 

1074 implementations of ``setDefaults`` must call their base class's 

1075 ``setDefaults``. 

1076 """ 

1077 pass 

1078 

1079 def update(self, **kw): 

1080 """Update values of fields specified by the keyword arguments. 

1081 

1082 Parameters 

1083 ---------- 

1084 kw 

1085 Keywords are configuration field names. Values are configuration 

1086 field values. 

1087 

1088 Notes 

1089 ----- 

1090 The ``__at`` and ``__label`` keyword arguments are special internal 

1091 keywords. They are used to strip out any internal steps from the 

1092 history tracebacks of the config. Do not modify these keywords to 

1093 subvert a `~lsst.pex.config.Config` instance's history. 

1094 

1095 Examples 

1096 -------- 

1097 This is a config with three fields: 

1098 

1099 >>> from lsst.pex.config import Config, Field 

1100 >>> class DemoConfig(Config): 

1101 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

1102 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

1103 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

1104 ... 

1105 >>> config = DemoConfig() 

1106 

1107 These are the default values of each field: 

1108 

1109 >>> for name, value in config.iteritems(): 

1110 ... print(f"{name}: {value}") 

1111 ... 

1112 fieldA: 42 

1113 fieldB: True 

1114 fieldC: 'Hello world' 

1115 

1116 Using this method to update ``fieldA`` and ``fieldC``: 

1117 

1118 >>> config.update(fieldA=13, fieldC='Updated!') 

1119 

1120 Now the values of each field are: 

1121 

1122 >>> for name, value in config.iteritems(): 

1123 ... print(f"{name}: {value}") 

1124 ... 

1125 fieldA: 13 

1126 fieldB: True 

1127 fieldC: 'Updated!' 

1128 """ 

1129 at = kw.pop("__at", getCallStack()) 

1130 label = kw.pop("__label", "update") 

1131 

1132 for name, value in kw.items(): 

1133 try: 

1134 field = self._fields[name] 

1135 field.__set__(self, value, at=at, label=label) 

1136 except KeyError: 

1137 raise KeyError(f"No field of name {name} exists in config type {_typeStr(self)}") 

1138 

1139 def load(self, filename, root="config"): 

1140 """Modify this config in place by executing the Python code in a 

1141 configuration file. 

1142 

1143 Parameters 

1144 ---------- 

1145 filename : `str` 

1146 Name of the configuration file. A configuration file is Python 

1147 module. 

1148 root : `str`, optional 

1149 Name of the variable in file that refers to the config being 

1150 overridden. 

1151 

1152 For example, the value of root is ``"config"`` and the file 

1153 contains:: 

1154 

1155 config.myField = 5 

1156 

1157 Then this config's field ``myField`` is set to ``5``. 

1158 

1159 See Also 

1160 -------- 

1161 lsst.pex.config.Config.loadFromStream 

1162 lsst.pex.config.Config.loadFromString 

1163 lsst.pex.config.Config.save 

1164 lsst.pex.config.Config.saveToStream 

1165 lsst.pex.config.Config.saveToString 

1166 """ 

1167 with open(filename) as f: 

1168 code = compile(f.read(), filename=filename, mode="exec") 

1169 self.loadFromString(code, root=root, filename=filename) 

1170 

1171 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None): 

1172 """Modify this Config in place by executing the Python code in the 

1173 provided stream. 

1174 

1175 Parameters 

1176 ---------- 

1177 stream : file-like object, `str`, `bytes`, or compiled string 

1178 Stream containing configuration override code. If this is a 

1179 code object, it should be compiled with ``mode="exec"``. 

1180 root : `str`, optional 

1181 Name of the variable in file that refers to the config being 

1182 overridden. 

1183 

1184 For example, the value of root is ``"config"`` and the file 

1185 contains:: 

1186 

1187 config.myField = 5 

1188 

1189 Then this config's field ``myField`` is set to ``5``. 

1190 filename : `str`, optional 

1191 Name of the configuration file, or `None` if unknown or contained 

1192 in the stream. Used for error reporting. 

1193 extraLocals : `dict` of `str` to `object`, optional 

1194 Any extra variables to include in local scope when loading. 

1195 

1196 Notes 

1197 ----- 

1198 For backwards compatibility reasons, this method accepts strings, bytes 

1199 and code objects as well as file-like objects. New code should use 

1200 `loadFromString` instead for most of these types. 

1201 

1202 See Also 

1203 -------- 

1204 lsst.pex.config.Config.load 

1205 lsst.pex.config.Config.loadFromString 

1206 lsst.pex.config.Config.save 

1207 lsst.pex.config.Config.saveToStream 

1208 lsst.pex.config.Config.saveToString 

1209 """ 

1210 if hasattr(stream, "read"): 1210 ↛ 1211line 1210 didn't jump to line 1211, because the condition on line 1210 was never true

1211 if filename is None: 

1212 filename = getattr(stream, "name", "?") 

1213 code = compile(stream.read(), filename=filename, mode="exec") 

1214 else: 

1215 code = stream 

1216 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals) 

1217 

1218 def loadFromString(self, code, root="config", filename=None, extraLocals=None): 

1219 """Modify this Config in place by executing the Python code in the 

1220 provided string. 

1221 

1222 Parameters 

1223 ---------- 

1224 code : `str`, `bytes`, or compiled string 

1225 Stream containing configuration override code. 

1226 root : `str`, optional 

1227 Name of the variable in file that refers to the config being 

1228 overridden. 

1229 

1230 For example, the value of root is ``"config"`` and the file 

1231 contains:: 

1232 

1233 config.myField = 5 

1234 

1235 Then this config's field ``myField`` is set to ``5``. 

1236 filename : `str`, optional 

1237 Name of the configuration file, or `None` if unknown or contained 

1238 in the stream. Used for error reporting. 

1239 extraLocals : `dict` of `str` to `object`, optional 

1240 Any extra variables to include in local scope when loading. 

1241 

1242 Raises 

1243 ------ 

1244 ValueError 

1245 Raised if a key in extraLocals is the same value as the value of 

1246 the root argument. 

1247 

1248 See Also 

1249 -------- 

1250 lsst.pex.config.Config.load 

1251 lsst.pex.config.Config.loadFromStream 

1252 lsst.pex.config.Config.save 

1253 lsst.pex.config.Config.saveToStream 

1254 lsst.pex.config.Config.saveToString 

1255 """ 

1256 if filename is None: 1256 ↛ 1260line 1256 didn't jump to line 1260, because the condition on line 1256 was never false

1257 # try to determine the file name; a compiled string 

1258 # has attribute "co_filename", 

1259 filename = getattr(code, "co_filename", "?") 

1260 with RecordingImporter() as importer: 

1261 globals = {"__file__": filename} 

1262 local = {root: self} 

1263 if extraLocals is not None: 1263 ↛ 1265line 1263 didn't jump to line 1265, because the condition on line 1263 was never true

1264 # verify the value of root was not passed as extra local args 

1265 if root in extraLocals: 

1266 raise ValueError( 

1267 f"{root} is reserved and cannot be used as a variable name in extraLocals" 

1268 ) 

1269 local.update(extraLocals) 

1270 exec(code, globals, local) 

1271 

1272 self._imports.update(importer.getModules()) 

1273 

1274 def save(self, filename, root="config"): 

1275 """Save a Python script to the named file, which, when loaded, 

1276 reproduces this config. 

1277 

1278 Parameters 

1279 ---------- 

1280 filename : `str` 

1281 Desination filename of this configuration. 

1282 root : `str`, optional 

1283 Name to use for the root config variable. The same value must be 

1284 used when loading (see `lsst.pex.config.Config.load`). 

1285 

1286 See Also 

1287 -------- 

1288 lsst.pex.config.Config.saveToStream 

1289 lsst.pex.config.Config.saveToString 

1290 lsst.pex.config.Config.load 

1291 lsst.pex.config.Config.loadFromStream 

1292 lsst.pex.config.Config.loadFromString 

1293 """ 

1294 d = os.path.dirname(filename) 

1295 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1296 self.saveToStream(outfile, root) 

1297 # tempfile is hardcoded to create files with mode '0600' 

1298 # for an explantion of these antics see: 

1299 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1300 umask = os.umask(0o077) 

1301 os.umask(umask) 

1302 os.chmod(outfile.name, (~umask & 0o666)) 

1303 # chmod before the move so we get quasi-atomic behavior if the 

1304 # source and dest. are on the same filesystem. 

1305 # os.rename may not work across filesystems 

1306 shutil.move(outfile.name, filename) 

1307 

1308 def saveToString(self, skipImports=False): 

1309 """Return the Python script form of this configuration as an executable 

1310 string. 

1311 

1312 Parameters 

1313 ---------- 

1314 skipImports : `bool`, optional 

1315 If `True` then do not include ``import`` statements in output, 

1316 this is to support human-oriented output from ``pipetask`` where 

1317 additional clutter is not useful. 

1318 

1319 Returns 

1320 ------- 

1321 code : `str` 

1322 A code string readable by `loadFromString`. 

1323 

1324 See Also 

1325 -------- 

1326 lsst.pex.config.Config.save 

1327 lsst.pex.config.Config.saveToStream 

1328 lsst.pex.config.Config.load 

1329 lsst.pex.config.Config.loadFromStream 

1330 lsst.pex.config.Config.loadFromString 

1331 """ 

1332 buffer = io.StringIO() 

1333 self.saveToStream(buffer, skipImports=skipImports) 

1334 return buffer.getvalue() 

1335 

1336 def saveToStream(self, outfile, root="config", skipImports=False): 

1337 """Save a configuration file to a stream, which, when loaded, 

1338 reproduces this config. 

1339 

1340 Parameters 

1341 ---------- 

1342 outfile : file-like object 

1343 Destination file object write the config into. Accepts strings not 

1344 bytes. 

1345 root 

1346 Name to use for the root config variable. The same value must be 

1347 used when loading (see `lsst.pex.config.Config.load`). 

1348 skipImports : `bool`, optional 

1349 If `True` then do not include ``import`` statements in output, 

1350 this is to support human-oriented output from ``pipetask`` where 

1351 additional clutter is not useful. 

1352 

1353 See Also 

1354 -------- 

1355 lsst.pex.config.Config.save 

1356 lsst.pex.config.Config.saveToString 

1357 lsst.pex.config.Config.load 

1358 lsst.pex.config.Config.loadFromStream 

1359 lsst.pex.config.Config.loadFromString 

1360 """ 

1361 tmp = self._name 

1362 self._rename(root) 

1363 try: 

1364 if not skipImports: 1364 ↛ 1378line 1364 didn't jump to line 1378, because the condition on line 1364 was never false

1365 self._collectImports() 

1366 # Remove self from the set, as it is handled explicitly below 

1367 self._imports.remove(self.__module__) 

1368 configType = type(self) 

1369 typeString = _typeStr(configType) 

1370 outfile.write(f"import {configType.__module__}\n") 

1371 outfile.write( 

1372 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1373 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n" 

1374 ) 

1375 for imp in sorted(self._imports): 1375 ↛ 1376line 1375 didn't jump to line 1376, because the loop on line 1375 never started

1376 if imp in sys.modules and sys.modules[imp] is not None: 

1377 outfile.write(f"import {imp}\n") 

1378 self._save(outfile) 

1379 finally: 

1380 self._rename(tmp) 

1381 

1382 def freeze(self): 

1383 """Make this config, and all subconfigs, read-only.""" 

1384 self._frozen = True 

1385 for field in self._fields.values(): 

1386 field.freeze(self) 

1387 

1388 def _save(self, outfile): 

1389 """Save this config to an open stream object. 

1390 

1391 Parameters 

1392 ---------- 

1393 outfile : file-like object 

1394 Destination file object write the config into. Accepts strings not 

1395 bytes. 

1396 """ 

1397 for field in self._fields.values(): 

1398 field.save(outfile, self) 

1399 

1400 def _collectImports(self): 

1401 """Add module containing self to the list of things to import and 

1402 then loops over all the fields in the config calling a corresponding 

1403 collect method. 

1404 

1405 The field method will call _collectImports on any 

1406 configs it may own and return the set of things to import. This 

1407 returned set will be merged with the set of imports for this config 

1408 class. 

1409 """ 

1410 self._imports.add(self.__module__) 

1411 for name, field in self._fields.items(): 

1412 field._collectImports(self, self._imports) 

1413 

1414 def toDict(self): 

1415 """Make a dictionary of field names and their values. 

1416 

1417 Returns 

1418 ------- 

1419 dict_ : `dict` 

1420 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1421 Values are `~lsst.pex.config.Field` values. 

1422 

1423 See Also 

1424 -------- 

1425 lsst.pex.config.Field.toDict 

1426 

1427 Notes 

1428 ----- 

1429 This method uses the `~lsst.pex.config.Field.toDict` method of 

1430 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1431 implement a ``toDict`` method for *this* method to work. 

1432 """ 

1433 dict_ = {} 

1434 for name, field in self._fields.items(): 

1435 dict_[name] = field.toDict(self) 

1436 return dict_ 

1437 

1438 def names(self): 

1439 """Get all the field names in the config, recursively. 

1440 

1441 Returns 

1442 ------- 

1443 names : `list` of `str` 

1444 Field names. 

1445 """ 

1446 # 

1447 # Rather than sort out the recursion all over again use the 

1448 # pre-existing saveToStream() 

1449 # 

1450 with io.StringIO() as strFd: 

1451 self.saveToStream(strFd, "config") 

1452 contents = strFd.getvalue() 

1453 strFd.close() 

1454 # 

1455 # Pull the names out of the dumped config 

1456 # 

1457 keys = [] 

1458 for line in contents.split("\n"): 

1459 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1460 continue 

1461 

1462 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1463 if mat: 

1464 keys.append(mat.group(1)) 

1465 

1466 return keys 

1467 

1468 def _rename(self, name): 

1469 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1470 

1471 Parameters 

1472 ---------- 

1473 name : `str` 

1474 New name for this config in its parent `~lsst.pex.config.Config`. 

1475 

1476 Notes 

1477 ----- 

1478 This method uses the `~lsst.pex.config.Field.rename` method of 

1479 individual `lsst.pex.config.Field` instances. 

1480 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1481 method for *this* method to work. 

1482 

1483 See Also 

1484 -------- 

1485 lsst.pex.config.Field.rename 

1486 """ 

1487 self._name = name 

1488 for field in self._fields.values(): 

1489 field.rename(self) 

1490 

1491 def validate(self): 

1492 """Validate the Config, raising an exception if invalid. 

1493 

1494 Raises 

1495 ------ 

1496 lsst.pex.config.FieldValidationError 

1497 Raised if verification fails. 

1498 

1499 Notes 

1500 ----- 

1501 The base class implementation performs type checks on all fields by 

1502 calling their `~lsst.pex.config.Field.validate` methods. 

1503 

1504 Complex single-field validation can be defined by deriving new Field 

1505 types. For convenience, some derived `lsst.pex.config.Field`-types 

1506 (`~lsst.pex.config.ConfigField` and 

1507 `~lsst.pex.config.ConfigChoiceField`) are defined in 

1508 ``lsst.pex.config`` that handle recursing into subconfigs. 

1509 

1510 Inter-field relationships should only be checked in derived 

1511 `~lsst.pex.config.Config` classes after calling this method, and base 

1512 validation is complete. 

1513 """ 

1514 for field in self._fields.values(): 

1515 field.validate(self) 

1516 

1517 def formatHistory(self, name, **kwargs): 

1518 """Format a configuration field's history to a human-readable format. 

1519 

1520 Parameters 

1521 ---------- 

1522 name : `str` 

1523 Name of a `~lsst.pex.config.Field` in this config. 

1524 kwargs 

1525 Keyword arguments passed to `lsst.pex.config.history.format`. 

1526 

1527 Returns 

1528 ------- 

1529 history : `str` 

1530 A string containing the formatted history. 

1531 

1532 See Also 

1533 -------- 

1534 lsst.pex.config.history.format 

1535 """ 

1536 import lsst.pex.config.history as pexHist 

1537 

1538 return pexHist.format(self, name, **kwargs) 

1539 

1540 history = property(lambda x: x._history) 1540 ↛ exitline 1540 didn't run the lambda on line 1540

1541 """Read-only history. 

1542 """ 

1543 

1544 def __setattr__(self, attr, value, at=None, label="assignment"): 

1545 """Set an attribute (such as a field's value). 

1546 

1547 Notes 

1548 ----- 

1549 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1550 locked such that no additional attributes nor properties may be added 

1551 to them dynamically. 

1552 

1553 Although this is not the standard Python behavior, it helps to protect 

1554 users from accidentally mispelling a field name, or trying to set a 

1555 non-existent field. 

1556 """ 

1557 if attr in self._fields: 

1558 if self._fields[attr].deprecated is not None: 1558 ↛ 1559line 1558 didn't jump to line 1559, because the condition on line 1558 was never true

1559 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1560 warnings.warn( 

1561 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1562 FutureWarning, 

1563 stacklevel=2, 

1564 ) 

1565 if at is None: 1565 ↛ 1568line 1565 didn't jump to line 1568, because the condition on line 1565 was never false

1566 at = getCallStack() 

1567 # This allows Field descriptors to work. 

1568 self._fields[attr].__set__(self, value, at=at, label=label) 

1569 elif hasattr(getattr(self.__class__, attr, None), "__set__"): 1569 ↛ 1571line 1569 didn't jump to line 1571, because the condition on line 1569 was never true

1570 # This allows properties and other non-Field descriptors to work. 

1571 return object.__setattr__(self, attr, value) 

1572 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1572 ↛ 1577line 1572 didn't jump to line 1577, because the condition on line 1572 was never false

1573 # This allows specific private attributes to work. 

1574 self.__dict__[attr] = value 

1575 else: 

1576 # We throw everything else. 

1577 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}") 

1578 

1579 def __delattr__(self, attr, at=None, label="deletion"): 

1580 if attr in self._fields: 

1581 if at is None: 

1582 at = getCallStack() 

1583 self._fields[attr].__delete__(self, at=at, label=label) 

1584 else: 

1585 object.__delattr__(self, attr) 

1586 

1587 def __eq__(self, other): 

1588 if type(other) == type(self): 1588 ↛ 1589line 1588 didn't jump to line 1589, because the condition on line 1588 was never true

1589 for name in self._fields: 

1590 thisValue = getattr(self, name) 

1591 otherValue = getattr(other, name) 

1592 if isinstance(thisValue, float) and math.isnan(thisValue): 

1593 if not math.isnan(otherValue): 

1594 return False 

1595 elif thisValue != otherValue: 

1596 return False 

1597 return True 

1598 return False 

1599 

1600 def __ne__(self, other): 

1601 return not self.__eq__(other) 

1602 

1603 def __str__(self): 

1604 return str(self.toDict()) 

1605 

1606 def __repr__(self): 

1607 return "{}({})".format( 

1608 _typeStr(self), 

1609 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None), 

1610 ) 

1611 

1612 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None): 

1613 """Compare this configuration to another `~lsst.pex.config.Config` for 

1614 equality. 

1615 

1616 Parameters 

1617 ---------- 

1618 other : `lsst.pex.config.Config` 

1619 Other `~lsst.pex.config.Config` object to compare against this 

1620 config. 

1621 shortcut : `bool`, optional 

1622 If `True`, return as soon as an inequality is found. Default is 

1623 `True`. 

1624 rtol : `float`, optional 

1625 Relative tolerance for floating point comparisons. 

1626 atol : `float`, optional 

1627 Absolute tolerance for floating point comparisons. 

1628 output : callable, optional 

1629 A callable that takes a string, used (possibly repeatedly) to 

1630 report inequalities. 

1631 

1632 Returns 

1633 ------- 

1634 isEqual : `bool` 

1635 `True` when the two `lsst.pex.config.Config` instances are equal. 

1636 `False` if there is an inequality. 

1637 

1638 See Also 

1639 -------- 

1640 lsst.pex.config.compareConfigs 

1641 

1642 Notes 

1643 ----- 

1644 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1645 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1646 are not considered by this method. 

1647 

1648 Floating point comparisons are performed by `numpy.allclose`. 

1649 """ 

1650 name1 = self._name if self._name is not None else "config" 

1651 name2 = other._name if other._name is not None else "config" 

1652 name = getComparisonName(name1, name2) 

1653 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output) 

1654 

1655 @classmethod 

1656 def __init_subclass__(cls, **kwargs): 

1657 """Run initialization for every subclass. 

1658 

1659 Specifically registers the subclass with a YAML representer 

1660 and YAML constructor (if pyyaml is available) 

1661 """ 

1662 super().__init_subclass__(**kwargs) 

1663 

1664 if not yaml: 1664 ↛ 1665line 1664 didn't jump to line 1665, because the condition on line 1664 was never true

1665 return 

1666 

1667 yaml.add_representer(cls, _yaml_config_representer) 

1668 

1669 @classmethod 

1670 def _fromPython(cls, config_py): 

1671 """Instantiate a `Config`-subclass from serialized Python form. 

1672 

1673 Parameters 

1674 ---------- 

1675 config_py : `str` 

1676 A serialized form of the Config as created by 

1677 `Config.saveToStream`. 

1678 

1679 Returns 

1680 ------- 

1681 config : `Config` 

1682 Reconstructed `Config` instant. 

1683 """ 

1684 cls = _classFromPython(config_py) 

1685 return unreduceConfig(cls, config_py) 

1686 

1687 

1688def _classFromPython(config_py): 

1689 """Return the Config subclass required by this Config serialization. 

1690 

1691 Parameters 

1692 ---------- 

1693 config_py : `str` 

1694 A serialized form of the Config as created by 

1695 `Config.saveToStream`. 

1696 

1697 Returns 

1698 ------- 

1699 cls : `type` 

1700 The `Config` subclass associated with this config. 

1701 """ 

1702 # standard serialization has the form: 

1703 # import config.class 

1704 # assert type(config)==config.class.Config, ... 

1705 # We want to parse these two lines so we can get the class itself 

1706 

1707 # Do a single regex to avoid large string copies when splitting a 

1708 # large config into separate lines. 

1709 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py) 

1710 

1711 if not matches: 

1712 first_line, second_line, _ = config_py.split("\n", 2) 

1713 raise ValueError( 

1714 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}" 

1715 ) 

1716 

1717 module_name = matches.group(1) 

1718 module = importlib.import_module(module_name) 

1719 

1720 # Second line 

1721 full_name = matches.group(2) 

1722 

1723 # Remove the module name from the full name 

1724 if not full_name.startswith(module_name): 

1725 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})") 

1726 

1727 # if module name is a.b.c and full name is a.b.c.d.E then 

1728 # we need to remove a.b.c. and iterate over the remainder 

1729 # The +1 is for the extra dot after a.b.c 

1730 remainder = full_name[len(module_name) + 1 :] 

1731 components = remainder.split(".") 

1732 pytype = module 

1733 for component in components: 

1734 pytype = getattr(pytype, component) 

1735 return pytype 

1736 

1737 

1738def unreduceConfig(cls, stream): 

1739 """Create a `~lsst.pex.config.Config` from a stream. 

1740 

1741 Parameters 

1742 ---------- 

1743 cls : `lsst.pex.config.Config`-type 

1744 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1745 with configurations in the ``stream``. 

1746 stream : file-like object, `str`, or compiled string 

1747 Stream containing configuration override code. 

1748 

1749 Returns 

1750 ------- 

1751 config : `lsst.pex.config.Config` 

1752 Config instance. 

1753 

1754 See Also 

1755 -------- 

1756 lsst.pex.config.Config.loadFromStream 

1757 """ 

1758 config = cls() 

1759 config.loadFromStream(stream) 

1760 return config