Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pex_config. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28__all__ = ("Config", "ConfigMeta", "Field", "FieldValidationError") 

29 

30import io 

31import os 

32import re 

33import sys 

34import math 

35import copy 

36import tempfile 

37import shutil 

38import warnings 

39 

40from .comparison import getComparisonName, compareScalars, compareConfigs 

41from .callStack import getStackFrame, getCallStack 

42 

43 

44def _joinNamePath(prefix=None, name=None, index=None): 

45 """Generate nested configuration names. 

46 """ 

47 if not prefix and not name: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 raise ValueError("Invalid name: cannot be None") 

49 elif not name: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true

50 name = prefix 

51 elif prefix and name: 51 ↛ 54line 51 didn't jump to line 54, because the condition on line 51 was never false

52 name = prefix + "." + name 

53 

54 if index is not None: 54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true

55 return "%s[%r]" % (name, index) 

56 else: 

57 return name 

58 

59 

60def _autocast(x, dtype): 

61 """Cast a value to a type, if appropriate. 

62 

63 Parameters 

64 ---------- 

65 x : object 

66 A value. 

67 dtype : tpye 

68 Data type, such as `float`, `int`, or `str`. 

69 

70 Returns 

71 ------- 

72 values : object 

73 If appropriate, the returned value is ``x`` cast to the given type 

74 ``dtype``. If the cast cannot be performed the original value of 

75 ``x`` is returned. 

76 """ 

77 if dtype == float and isinstance(x, int): 

78 return float(x) 

79 return x 

80 

81 

82def _typeStr(x): 

83 """Generate a fully-qualified type name. 

84 

85 Returns 

86 ------- 

87 `str` 

88 Fully-qualified type name. 

89 

90 Notes 

91 ----- 

92 This function is used primarily for writing config files to be executed 

93 later upon with the 'load' function. 

94 """ 

95 if hasattr(x, '__module__') and hasattr(x, '__name__'): 

96 xtype = x 

97 else: 

98 xtype = type(x) 

99 if (sys.version_info.major <= 2 and xtype.__module__ == '__builtin__') or xtype.__module__ == 'builtins': 99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true

100 return xtype.__name__ 

101 else: 

102 return "%s.%s" % (xtype.__module__, xtype.__name__) 

103 

104 

105class ConfigMeta(type): 

106 """A metaclass for `lsst.pex.config.Config`. 

107 

108 Notes 

109 ----- 

110 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field` 

111 class attributes as a class attribute called ``_fields``, and adds 

112 the name of each field as an instance variable of the field itself (so you 

113 don't have to pass the name of the field to the field constructor). 

114 """ 

115 

116 def __init__(cls, name, bases, dict_): 

117 type.__init__(cls, name, bases, dict_) 

118 cls._fields = {} 

119 cls._source = getStackFrame() 

120 

121 def getFields(classtype): 

122 fields = {} 

123 bases = list(classtype.__bases__) 

124 bases.reverse() 

125 for b in bases: 

126 fields.update(getFields(b)) 

127 

128 for k, v in classtype.__dict__.items(): 

129 if isinstance(v, Field): 

130 fields[k] = v 

131 return fields 

132 

133 fields = getFields(cls) 

134 for k, v in fields.items(): 

135 setattr(cls, k, copy.deepcopy(v)) 

136 

137 def __setattr__(cls, name, value): 

138 if isinstance(value, Field): 

139 value.name = name 

140 cls._fields[name] = value 

141 type.__setattr__(cls, name, value) 

142 

143 

144class FieldValidationError(ValueError): 

145 """Raised when a ``~lsst.pex.config.Field`` is not valid in a 

146 particular ``~lsst.pex.config.Config``. 

147 

148 Parameters 

149 ---------- 

150 field : `lsst.pex.config.Field` 

151 The field that was not valid. 

152 config : `lsst.pex.config.Config` 

153 The config containing the invalid field. 

154 msg : `str` 

155 Text describing why the field was not valid. 

156 """ 

157 

158 def __init__(self, field, config, msg): 

159 self.fieldType = type(field) 

160 """Type of the `~lsst.pex.config.Field` that incurred the error. 

161 """ 

162 

163 self.fieldName = field.name 

164 """Name of the `~lsst.pex.config.Field` instance that incurred the 

165 error (`str`). 

166 

167 See also 

168 -------- 

169 lsst.pex.config.Field.name 

170 """ 

171 

172 self.fullname = _joinNamePath(config._name, field.name) 

173 """Fully-qualified name of the `~lsst.pex.config.Field` instance 

174 (`str`). 

175 """ 

176 

177 self.history = config.history.setdefault(field.name, []) 

178 """Full history of all changes to the `~lsst.pex.config.Field` 

179 instance. 

180 """ 

181 

182 self.fieldSource = field.source 

183 """File and line number of the `~lsst.pex.config.Field` definition. 

184 """ 

185 

186 self.configSource = config._source 

187 error = "%s '%s' failed validation: %s\n"\ 

188 "For more information see the Field definition at:\n%s"\ 

189 " and the Config definition at:\n%s" % \ 

190 (self.fieldType.__name__, self.fullname, msg, 

191 self.fieldSource.format(), self.configSource.format()) 

192 super().__init__(error) 

193 

194 

195class Field: 

196 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`, 

197 `complex`, `bool`, and `str` data types. 

198 

199 Parameters 

200 ---------- 

201 doc : `str` 

202 A description of the field for users. 

203 dtype : type 

204 The field's data type. ``Field`` only supports basic data types: 

205 `int`, `float`, `complex`, `bool`, and `str`. See 

206 `Field.supportedTypes`. 

207 default : object, optional 

208 The field's default value. 

209 check : callable, optional 

210 A callable that is called with the field's value. This callable should 

211 return `False` if the value is invalid. More complex inter-field 

212 validation can be written as part of the 

213 `lsst.pex.config.Config.validate` method. 

214 optional : `bool`, optional 

215 This sets whether the field is considered optional, and therefore 

216 doesn't need to be set by the user. When `False`, 

217 `lsst.pex.config.Config.validate` fails if the field's value is `None`. 

218 deprecated : None or `str`, optional 

219 A description of why this Field is deprecated, including removal date. 

220 If not None, the string is appended to the docstring for this Field. 

221 

222 Raises 

223 ------ 

224 ValueError 

225 Raised when the ``dtype`` parameter is not one of the supported types 

226 (see `Field.supportedTypes`). 

227 

228 See also 

229 -------- 

230 ChoiceField 

231 ConfigChoiceField 

232 ConfigDictField 

233 ConfigField 

234 ConfigurableField 

235 DictField 

236 ListField 

237 RangeField 

238 RegistryField 

239 

240 Notes 

241 ----- 

242 ``Field`` instances (including those of any subclass of ``Field``) are used 

243 as class attributes of `~lsst.pex.config.Config` subclasses (see the 

244 example, below). ``Field`` attributes work like the `property` attributes 

245 of classes that implement custom setters and getters. `Field` attributes 

246 belong to the class, but operate on the instance. Formally speaking, 

247 `Field` attributes are `descriptors 

248 <https://docs.python.org/3/howto/descriptor.html>`_. 

249 

250 When you access a `Field` attribute on a `Config` instance, you don't 

251 get the `Field` instance itself. Instead, you get the value of that field, 

252 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom 

253 container type (like a `lsst.pex.config.List`) depending on the field's 

254 type. See the example, below. 

255 

256 Examples 

257 -------- 

258 Instances of ``Field`` should be used as class attributes of 

259 `lsst.pex.config.Config` subclasses: 

260 

261 >>> from lsst.pex.config import Config, Field 

262 >>> class Example(Config): 

263 ... myInt = Field("An integer field.", int, default=0) 

264 ... 

265 >>> print(config.myInt) 

266 0 

267 >>> config.myInt = 5 

268 >>> print(config.myInt) 

269 5 

270 """ 

271 

272 supportedTypes = set((str, bool, float, int, complex)) 

273 """Supported data types for field values (`set` of types). 

274 """ 

275 

276 def __init__(self, doc, dtype, default=None, check=None, optional=False, deprecated=None): 

277 if dtype not in self.supportedTypes: 277 ↛ 278line 277 didn't jump to line 278, because the condition on line 277 was never true

278 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) 

279 

280 source = getStackFrame() 

281 self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source, 

282 deprecated=deprecated) 

283 

284 def _setup(self, doc, dtype, default, check, optional, source, deprecated): 

285 """Set attributes, usually during initialization. 

286 """ 

287 self.dtype = dtype 

288 """Data type for the field. 

289 """ 

290 

291 # append the deprecation message to the docstring. 

292 if deprecated is not None: 

293 doc = f"{doc} Deprecated: {deprecated}" 

294 self.doc = doc 

295 """A description of the field (`str`). 

296 """ 

297 

298 self.deprecated = deprecated 

299 """If not None, a description of why this field is deprecated (`str`). 

300 """ 

301 

302 self.__doc__ = f"{doc} (`{dtype.__name__}`" 

303 if optional or default is not None: 

304 self.__doc__ += f", default ``{default!r}``" 

305 self.__doc__ += ")" 

306 

307 self.default = default 

308 """Default value for this field. 

309 """ 

310 

311 self.check = check 

312 """A user-defined function that validates the value of the field. 

313 """ 

314 

315 self.optional = optional 

316 """Flag that determines if the field is required to be set (`bool`). 

317 

318 When `False`, `lsst.pex.config.Config.validate` will fail if the 

319 field's value is `None`. 

320 """ 

321 

322 self.source = source 

323 """The stack frame where this field is defined (`list` of 

324 `lsst.pex.config.callStack.StackFrame`). 

325 """ 

326 

327 def rename(self, instance): 

328 """Rename the field in a `~lsst.pex.config.Config` (for internal use 

329 only). 

330 

331 Parameters 

332 ---------- 

333 instance : `lsst.pex.config.Config` 

334 The config instance that contains this field. 

335 

336 Notes 

337 ----- 

338 This method is invoked by the `lsst.pex.config.Config` object that 

339 contains this field and should not be called directly. 

340 

341 Renaming is only relevant for `~lsst.pex.config.Field` instances that 

342 hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should 

343 rename each subconfig with the full field name as generated by 

344 `lsst.pex.config.config._joinNamePath`. 

345 """ 

346 pass 

347 

348 def validate(self, instance): 

349 """Validate the field (for internal use only). 

350 

351 Parameters 

352 ---------- 

353 instance : `lsst.pex.config.Config` 

354 The config instance that contains this field. 

355 

356 Raises 

357 ------ 

358 lsst.pex.config.FieldValidationError 

359 Raised if verification fails. 

360 

361 Notes 

362 ----- 

363 This method provides basic validation: 

364 

365 - Ensures that the value is not `None` if the field is not optional. 

366 - Ensures type correctness. 

367 - Ensures that the user-provided ``check`` function is valid. 

368 

369 Most `~lsst.pex.config.Field` subclasses should call 

370 `lsst.pex.config.field.Field.validate` if they re-implement 

371 `~lsst.pex.config.field.Field.validate`. 

372 """ 

373 value = self.__get__(instance) 

374 if not self.optional and value is None: 

375 raise FieldValidationError(self, instance, "Required value cannot be None") 

376 

377 def freeze(self, instance): 

378 """Make this field read-only (for internal use only). 

379 

380 Parameters 

381 ---------- 

382 instance : `lsst.pex.config.Config` 

383 The config instance that contains this field. 

384 

385 Notes 

386 ----- 

387 Freezing is only relevant for fields that hold subconfigs. Fields which 

388 hold subconfigs should freeze each subconfig. 

389 

390 **Subclasses should implement this method.** 

391 """ 

392 pass 

393 

394 def _validateValue(self, value): 

395 """Validate a value. 

396 

397 Parameters 

398 ---------- 

399 value : object 

400 The value being validated. 

401 

402 Raises 

403 ------ 

404 TypeError 

405 Raised if the value's type is incompatible with the field's 

406 ``dtype``. 

407 ValueError 

408 Raised if the value is rejected by the ``check`` method. 

409 """ 

410 if value is None: 410 ↛ 411line 410 didn't jump to line 411, because the condition on line 410 was never true

411 return 

412 

413 if not isinstance(value, self.dtype): 413 ↛ 414line 413 didn't jump to line 414, because the condition on line 413 was never true

414 msg = "Value %s is of incorrect type %s. Expected type %s" % \ 

415 (value, _typeStr(value), _typeStr(self.dtype)) 

416 raise TypeError(msg) 

417 if self.check is not None and not self.check(value): 417 ↛ 418line 417 didn't jump to line 418, because the condition on line 417 was never true

418 msg = "Value %s is not a valid value" % str(value) 

419 raise ValueError(msg) 

420 

421 def _collectImports(self, instance, imports): 

422 """This function should call the _collectImports method on all config 

423 objects the field may own, and union them with the supplied imports 

424 set. 

425 

426 Parameters 

427 ---------- 

428 instance : instance or subclass of `lsst.pex.config.Config` 

429 A config object that has this field defined on it 

430 imports : `set` 

431 Set of python modules that need imported after persistence 

432 """ 

433 pass 

434 

435 def save(self, outfile, instance): 

436 """Save this field to a file (for internal use only). 

437 

438 Parameters 

439 ---------- 

440 outfile : file-like object 

441 A writeable field handle. 

442 instance : `Config` 

443 The `Config` instance that contains this field. 

444 

445 Notes 

446 ----- 

447 This method is invoked by the `~lsst.pex.config.Config` object that 

448 contains this field and should not be called directly. 

449 

450 The output consists of the documentation string 

451 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second 

452 line is formatted as an assignment: ``{fullname}={value}``. 

453 

454 This output can be executed with Python. 

455 """ 

456 value = self.__get__(instance) 

457 fullname = _joinNamePath(instance._name, self.name) 

458 

459 if self.deprecated and value == self.default: 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true

460 return 

461 

462 # write full documentation string as comment lines 

463 # (i.e. first character is #) 

464 doc = "# " + str(self.doc).replace("\n", "\n# ") 

465 if isinstance(value, float) and (math.isinf(value) or math.isnan(value)): 465 ↛ 467line 465 didn't jump to line 467, because the condition on line 465 was never true

466 # non-finite numbers need special care 

467 outfile.write(u"{}\n{}=float('{!r}')\n\n".format(doc, fullname, value)) 

468 else: 

469 outfile.write(u"{}\n{}={!r}\n\n".format(doc, fullname, value)) 

470 

471 def toDict(self, instance): 

472 """Convert the field value so that it can be set as the value of an 

473 item in a `dict` (for internal use only). 

474 

475 Parameters 

476 ---------- 

477 instance : `Config` 

478 The `Config` that contains this field. 

479 

480 Returns 

481 ------- 

482 value : object 

483 The field's value. See *Notes*. 

484 

485 Notes 

486 ----- 

487 This method invoked by the owning `~lsst.pex.config.Config` object and 

488 should not be called directly. 

489 

490 Simple values are passed through. Complex data structures must be 

491 manipulated. For example, a `~lsst.pex.config.Field` holding a 

492 subconfig should, instead of the subconfig object, return a `dict` 

493 where the keys are the field names in the subconfig, and the values are 

494 the field values in the subconfig. 

495 """ 

496 return self.__get__(instance) 

497 

498 def __get__(self, instance, owner=None, at=None, label="default"): 

499 """Define how attribute access should occur on the Config instance 

500 This is invoked by the owning config object and should not be called 

501 directly 

502 

503 When the field attribute is accessed on a Config class object, it 

504 returns the field object itself in order to allow inspection of 

505 Config classes. 

506 

507 When the field attribute is access on a config instance, the actual 

508 value described by the field (and held by the Config instance) is 

509 returned. 

510 """ 

511 if instance is None or not isinstance(instance, Config): 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true

512 return self 

513 else: 

514 return instance._storage[self.name] 

515 

516 def __set__(self, instance, value, at=None, label='assignment'): 

517 """Set an attribute on the config instance. 

518 

519 Parameters 

520 ---------- 

521 instance : `lsst.pex.config.Config` 

522 The config instance that contains this field. 

523 value : obj 

524 Value to set on this field. 

525 at : `list` of `lsst.pex.config.callStack.StackFrame` 

526 The call stack (created by 

527 `lsst.pex.config.callStack.getCallStack`). 

528 label : `str`, optional 

529 Event label for the history. 

530 

531 Notes 

532 ----- 

533 This method is invoked by the owning `lsst.pex.config.Config` object 

534 and should not be called directly. 

535 

536 Derived `~lsst.pex.config.Field` classes may need to override the 

537 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors 

538 should follow the following rules: 

539 

540 - Do not allow modification of frozen configs. 

541 - Validate the new value **before** modifying the field. Except if the 

542 new value is `None`. `None` is special and no attempt should be made 

543 to validate it until `lsst.pex.config.Config.validate` is called. 

544 - Do not modify the `~lsst.pex.config.Config` instance to contain 

545 invalid values. 

546 - If the field is modified, update the history of the 

547 `lsst.pex.config.field.Field` to reflect the changes. 

548 

549 In order to decrease the need to implement this method in derived 

550 `~lsst.pex.config.Field` types, value validation is performed in the 

551 `lsst.pex.config.Field._validateValue`. If only the validation step 

552 differs in the derived `~lsst.pex.config.Field`, it is simpler to 

553 implement `lsst.pex.config.Field._validateValue` than to reimplement 

554 ``__set__``. More complicated behavior, however, may require 

555 reimplementation. 

556 """ 

557 if instance._frozen: 557 ↛ 558line 557 didn't jump to line 558, because the condition on line 557 was never true

558 raise FieldValidationError(self, instance, "Cannot modify a frozen Config") 

559 

560 history = instance._history.setdefault(self.name, []) 

561 if value is not None: 561 ↛ 568line 561 didn't jump to line 568, because the condition on line 561 was never false

562 value = _autocast(value, self.dtype) 

563 try: 

564 self._validateValue(value) 

565 except BaseException as e: 

566 raise FieldValidationError(self, instance, str(e)) 

567 

568 instance._storage[self.name] = value 

569 if at is None: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true

570 at = getCallStack() 

571 history.append((value, at, label)) 

572 

573 def __delete__(self, instance, at=None, label='deletion'): 

574 """Delete an attribute from a `lsst.pex.config.Config` instance. 

575 

576 Parameters 

577 ---------- 

578 instance : `lsst.pex.config.Config` 

579 The config instance that contains this field. 

580 at : `list` of `lsst.pex.config.callStack.StackFrame` 

581 The call stack (created by 

582 `lsst.pex.config.callStack.getCallStack`). 

583 label : `str`, optional 

584 Event label for the history. 

585 

586 Notes 

587 ----- 

588 This is invoked by the owning `~lsst.pex.config.Config` object and 

589 should not be called directly. 

590 """ 

591 if at is None: 

592 at = getCallStack() 

593 self.__set__(instance, None, at=at, label=label) 

594 

595 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

596 """Compare a field (named `Field.name`) in two 

597 `~lsst.pex.config.Config` instances for equality. 

598 

599 Parameters 

600 ---------- 

601 instance1 : `lsst.pex.config.Config` 

602 Left-hand side `Config` instance to compare. 

603 instance2 : `lsst.pex.config.Config` 

604 Right-hand side `Config` instance to compare. 

605 shortcut : `bool`, optional 

606 **Unused.** 

607 rtol : `float`, optional 

608 Relative tolerance for floating point comparisons. 

609 atol : `float`, optional 

610 Absolute tolerance for floating point comparisons. 

611 output : callable, optional 

612 A callable that takes a string, used (possibly repeatedly) to 

613 report inequalities. 

614 

615 Notes 

616 ----- 

617 This method must be overridden by more complex `Field` subclasses. 

618 

619 See also 

620 -------- 

621 lsst.pex.config.compareScalars 

622 """ 

623 v1 = getattr(instance1, self.name) 

624 v2 = getattr(instance2, self.name) 

625 name = getComparisonName( 

626 _joinNamePath(instance1._name, self.name), 

627 _joinNamePath(instance2._name, self.name) 

628 ) 

629 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output) 

630 

631 

632class RecordingImporter: 

633 """Importer (for `sys.meta_path`) that records which modules are being 

634 imported. 

635 

636 *This class does not do any importing itself.* 

637 

638 Examples 

639 -------- 

640 Use this class as a context manager to ensure it is properly uninstalled 

641 when done: 

642 

643 >>> with RecordingImporter() as importer: 

644 ... # import stuff 

645 ... import numpy as np 

646 ... print("Imported: " + importer.getModules()) 

647 """ 

648 

649 def __init__(self): 

650 self._modules = set() 

651 

652 def __enter__(self): 

653 self.origMetaPath = sys.meta_path 

654 sys.meta_path = [self] + sys.meta_path 

655 return self 

656 

657 def __exit__(self, *args): 

658 self.uninstall() 

659 return False # Don't suppress exceptions 

660 

661 def uninstall(self): 

662 """Uninstall the importer. 

663 """ 

664 sys.meta_path = self.origMetaPath 

665 

666 def find_module(self, fullname, path=None): 

667 """Called as part of the ``import`` chain of events. 

668 """ 

669 self._modules.add(fullname) 

670 # Return None because we don't do any importing. 

671 return None 

672 

673 def getModules(self): 

674 """Get the set of modules that were imported. 

675 

676 Returns 

677 ------- 

678 modules : `set` of `str` 

679 Set of imported module names. 

680 """ 

681 return self._modules 

682 

683 

684class Config(metaclass=ConfigMeta): 

685 """Base class for configuration (*config*) objects. 

686 

687 Notes 

688 ----- 

689 A ``Config`` object will usually have several `~lsst.pex.config.Field` 

690 instances as class attributes. These are used to define most of the base 

691 class behavior. 

692 

693 ``Config`` implements a mapping API that provides many `dict`-like methods, 

694 such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and 

695 `itervalues`. ``Config`` instances also support the ``in`` operator to 

696 test if a field is in the config. Unlike a `dict`, ``Config`` classes are 

697 not subscriptable. Instead, access individual fields as attributes of the 

698 configuration instance. 

699 

700 Examples 

701 -------- 

702 Config classes are subclasses of ``Config`` that have 

703 `~lsst.pex.config.Field` instances (or instances of 

704 `~lsst.pex.config.Field` subclasses) as class attributes: 

705 

706 >>> from lsst.pex.config import Config, Field, ListField 

707 >>> class DemoConfig(Config): 

708 ... intField = Field(doc="An integer field", dtype=int, default=42) 

709 ... listField = ListField(doc="List of favorite beverages.", dtype=str, 

710 ... default=['coffee', 'green tea', 'water']) 

711 ... 

712 >>> config = DemoConfig() 

713 

714 Configs support many `dict`-like APIs: 

715 

716 >>> config.keys() 

717 ['intField', 'listField'] 

718 >>> 'intField' in config 

719 True 

720 

721 Individual fields can be accessed as attributes of the configuration: 

722 

723 >>> config.intField 

724 42 

725 >>> config.listField.append('earl grey tea') 

726 >>> print(config.listField) 

727 ['coffee', 'green tea', 'water', 'earl grey tea'] 

728 """ 

729 

730 def __iter__(self): 

731 """Iterate over fields. 

732 """ 

733 return self._fields.__iter__() 

734 

735 def keys(self): 

736 """Get field names. 

737 

738 Returns 

739 ------- 

740 names : `list` 

741 List of `lsst.pex.config.Field` names. 

742 

743 See also 

744 -------- 

745 lsst.pex.config.Config.iterkeys 

746 """ 

747 return list(self._storage.keys()) 

748 

749 def values(self): 

750 """Get field values. 

751 

752 Returns 

753 ------- 

754 values : `list` 

755 List of field values. 

756 

757 See also 

758 -------- 

759 lsst.pex.config.Config.itervalues 

760 """ 

761 return list(self._storage.values()) 

762 

763 def items(self): 

764 """Get configurations as ``(field name, field value)`` pairs. 

765 

766 Returns 

767 ------- 

768 items : `list` 

769 List of tuples for each configuration. Tuple items are: 

770 

771 0. Field name. 

772 1. Field value. 

773 

774 See also 

775 -------- 

776 lsst.pex.config.Config.iteritems 

777 """ 

778 return list(self._storage.items()) 

779 

780 def iteritems(self): 

781 """Iterate over (field name, field value) pairs. 

782 

783 Yields 

784 ------ 

785 item : `tuple` 

786 Tuple items are: 

787 

788 0. Field name. 

789 1. Field value. 

790 

791 See also 

792 -------- 

793 lsst.pex.config.Config.items 

794 """ 

795 return iter(self._storage.items()) 

796 

797 def itervalues(self): 

798 """Iterate over field values. 

799 

800 Yields 

801 ------ 

802 value : obj 

803 A field value. 

804 

805 See also 

806 -------- 

807 lsst.pex.config.Config.values 

808 """ 

809 return iter(self.storage.values()) 

810 

811 def iterkeys(self): 

812 """Iterate over field names 

813 

814 Yields 

815 ------ 

816 key : `str` 

817 A field's key (attribute name). 

818 

819 See also 

820 -------- 

821 lsst.pex.config.Config.values 

822 """ 

823 return iter(self.storage.keys()) 

824 

825 def __contains__(self, name): 

826 """!Return True if the specified field exists in this config 

827 

828 @param[in] name field name to test for 

829 """ 

830 return self._storage.__contains__(name) 

831 

832 def __new__(cls, *args, **kw): 

833 """Allocate a new `lsst.pex.config.Config` object. 

834 

835 In order to ensure that all Config object are always in a proper state 

836 when handed to users or to derived `~lsst.pex.config.Config` classes, 

837 some attributes are handled at allocation time rather than at 

838 initialization. 

839 

840 This ensures that even if a derived `~lsst.pex.config.Config` class 

841 implements ``__init__``, its author does not need to be concerned about 

842 when or even the base ``Config.__init__`` should be called. 

843 """ 

844 name = kw.pop("__name", None) 

845 at = kw.pop("__at", getCallStack()) 

846 # remove __label and ignore it 

847 kw.pop("__label", "default") 

848 

849 instance = object.__new__(cls) 

850 instance._frozen = False 

851 instance._name = name 

852 instance._storage = {} 

853 instance._history = {} 

854 instance._imports = set() 

855 # load up defaults 

856 for field in instance._fields.values(): 

857 instance._history[field.name] = [] 

858 field.__set__(instance, field.default, at=at + [field.source], label="default") 

859 # set custom default-overides 

860 instance.setDefaults() 

861 # set constructor overides 

862 instance.update(__at=at, **kw) 

863 return instance 

864 

865 def __reduce__(self): 

866 """Reduction for pickling (function with arguments to reproduce). 

867 

868 We need to condense and reconstitute the `~lsst.pex.config.Config`, 

869 since it may contain lambdas (as the ``check`` elements) that cannot 

870 be pickled. 

871 """ 

872 # The stream must be in characters to match the API but pickle 

873 # requires bytes 

874 stream = io.StringIO() 

875 self.saveToStream(stream) 

876 return (unreduceConfig, (self.__class__, stream.getvalue().encode())) 

877 

878 def setDefaults(self): 

879 """Subclass hook for computing defaults. 

880 

881 Notes 

882 ----- 

883 Derived `~lsst.pex.config.Config` classes that must compute defaults 

884 rather than using the `~lsst.pex.config.Field` instances's defaults 

885 should do so here. To correctly use inherited defaults, 

886 implementations of ``setDefaults`` must call their base class's 

887 ``setDefaults``. 

888 """ 

889 pass 

890 

891 def update(self, **kw): 

892 """Update values of fields specified by the keyword arguments. 

893 

894 Parameters 

895 ---------- 

896 kw 

897 Keywords are configuration field names. Values are configuration 

898 field values. 

899 

900 Notes 

901 ----- 

902 The ``__at`` and ``__label`` keyword arguments are special internal 

903 keywords. They are used to strip out any internal steps from the 

904 history tracebacks of the config. Do not modify these keywords to 

905 subvert a `~lsst.pex.config.Config` instance's history. 

906 

907 Examples 

908 -------- 

909 This is a config with three fields: 

910 

911 >>> from lsst.pex.config import Config, Field 

912 >>> class DemoConfig(Config): 

913 ... fieldA = Field(doc='Field A', dtype=int, default=42) 

914 ... fieldB = Field(doc='Field B', dtype=bool, default=True) 

915 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world') 

916 ... 

917 >>> config = DemoConfig() 

918 

919 These are the default values of each field: 

920 

921 >>> for name, value in config.iteritems(): 

922 ... print(f"{name}: {value}") 

923 ... 

924 fieldA: 42 

925 fieldB: True 

926 fieldC: 'Hello world' 

927 

928 Using this method to update ``fieldA`` and ``fieldC``: 

929 

930 >>> config.update(fieldA=13, fieldC='Updated!') 

931 

932 Now the values of each field are: 

933 

934 >>> for name, value in config.iteritems(): 

935 ... print(f"{name}: {value}") 

936 ... 

937 fieldA: 13 

938 fieldB: True 

939 fieldC: 'Updated!' 

940 """ 

941 at = kw.pop("__at", getCallStack()) 

942 label = kw.pop("__label", "update") 

943 

944 for name, value in kw.items(): 

945 try: 

946 field = self._fields[name] 

947 field.__set__(self, value, at=at, label=label) 

948 except KeyError: 

949 raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) 

950 

951 def load(self, filename, root="config"): 

952 """Modify this config in place by executing the Python code in a 

953 configuration file. 

954 

955 Parameters 

956 ---------- 

957 filename : `str` 

958 Name of the configuration file. A configuration file is Python 

959 module. 

960 root : `str`, optional 

961 Name of the variable in file that refers to the config being 

962 overridden. 

963 

964 For example, the value of root is ``"config"`` and the file 

965 contains:: 

966 

967 config.myField = 5 

968 

969 Then this config's field ``myField`` is set to ``5``. 

970 

971 **Deprecated:** For backwards compatibility, older config files 

972 that use ``root="root"`` instead of ``root="config"`` will be 

973 loaded with a warning printed to `sys.stderr`. This feature will be 

974 removed at some point. 

975 

976 See also 

977 -------- 

978 lsst.pex.config.Config.loadFromStream 

979 lsst.pex.config.Config.save 

980 lsst.pex.config.Config.saveFromStream 

981 """ 

982 with open(filename, "r") as f: 

983 code = compile(f.read(), filename=filename, mode="exec") 

984 self.loadFromStream(stream=code, root=root, filename=filename) 

985 

986 def loadFromStream(self, stream, root="config", filename=None): 

987 """Modify this Config in place by executing the Python code in the 

988 provided stream. 

989 

990 Parameters 

991 ---------- 

992 stream : file-like object, `str`, or compiled string 

993 Stream containing configuration override code. 

994 root : `str`, optional 

995 Name of the variable in file that refers to the config being 

996 overridden. 

997 

998 For example, the value of root is ``"config"`` and the file 

999 contains:: 

1000 

1001 config.myField = 5 

1002 

1003 Then this config's field ``myField`` is set to ``5``. 

1004 

1005 **Deprecated:** For backwards compatibility, older config files 

1006 that use ``root="root"`` instead of ``root="config"`` will be 

1007 loaded with a warning printed to `sys.stderr`. This feature will be 

1008 removed at some point. 

1009 filename : `str`, optional 

1010 Name of the configuration file, or `None` if unknown or contained 

1011 in the stream. Used for error reporting. 

1012 

1013 See also 

1014 -------- 

1015 lsst.pex.config.Config.load 

1016 lsst.pex.config.Config.save 

1017 lsst.pex.config.Config.saveFromStream 

1018 """ 

1019 with RecordingImporter() as importer: 

1020 globals = {"__file__": filename} 

1021 try: 

1022 local = {root: self} 

1023 exec(stream, globals, local) 

1024 except NameError as e: 

1025 if root == "config" and "root" in e.args[0]: 

1026 if filename is None: 

1027 # try to determine the file name; a compiled string 

1028 # has attribute "co_filename", 

1029 # an open file has attribute "name", else give up 

1030 filename = getattr(stream, "co_filename", None) 

1031 if filename is None: 

1032 filename = getattr(stream, "name", "?") 

1033 print(f"Config override file {filename!r}" 

1034 " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr) 

1035 local = {"root": self} 

1036 exec(stream, globals, local) 

1037 else: 

1038 raise 

1039 

1040 self._imports.update(importer.getModules()) 

1041 

1042 def save(self, filename, root="config"): 

1043 """Save a Python script to the named file, which, when loaded, 

1044 reproduces this config. 

1045 

1046 Parameters 

1047 ---------- 

1048 filename : `str` 

1049 Desination filename of this configuration. 

1050 root : `str`, optional 

1051 Name to use for the root config variable. The same value must be 

1052 used when loading (see `lsst.pex.config.Config.load`). 

1053 

1054 See also 

1055 -------- 

1056 lsst.pex.config.Config.saveToStream 

1057 lsst.pex.config.Config.load 

1058 lsst.pex.config.Config.loadFromStream 

1059 """ 

1060 d = os.path.dirname(filename) 

1061 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile: 

1062 self.saveToStream(outfile, root) 

1063 # tempfile is hardcoded to create files with mode '0600' 

1064 # for an explantion of these antics see: 

1065 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python 

1066 umask = os.umask(0o077) 

1067 os.umask(umask) 

1068 os.chmod(outfile.name, (~umask & 0o666)) 

1069 # chmod before the move so we get quasi-atomic behavior if the 

1070 # source and dest. are on the same filesystem. 

1071 # os.rename may not work across filesystems 

1072 shutil.move(outfile.name, filename) 

1073 

1074 def saveToStream(self, outfile, root="config", skipImports=False): 

1075 """Save a configuration file to a stream, which, when loaded, 

1076 reproduces this config. 

1077 

1078 Parameters 

1079 ---------- 

1080 outfile : file-like object 

1081 Destination file object write the config into. Accepts strings not 

1082 bytes. 

1083 root 

1084 Name to use for the root config variable. The same value must be 

1085 used when loading (see `lsst.pex.config.Config.load`). 

1086 skipImports : `bool`, optional 

1087 If `True` then do not include ``import`` statements in output, 

1088 this is to support human-oriented output from ``pipetask`` where 

1089 additional clutter is not useful. 

1090 

1091 See also 

1092 -------- 

1093 lsst.pex.config.Config.save 

1094 lsst.pex.config.Config.load 

1095 lsst.pex.config.Config.loadFromStream 

1096 """ 

1097 tmp = self._name 

1098 self._rename(root) 

1099 try: 

1100 if not skipImports: 1100 ↛ 1112line 1100 didn't jump to line 1112, because the condition on line 1100 was never false

1101 self._collectImports() 

1102 # Remove self from the set, as it is handled explicitly below 

1103 self._imports.remove(self.__module__) 

1104 configType = type(self) 

1105 typeString = _typeStr(configType) 

1106 outfile.write(f"import {configType.__module__}\n") 

1107 outfile.write(f"assert type({root})=={typeString}, 'config is of type %s.%s instead of " 

1108 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n") 

1109 for imp in self._imports: 1109 ↛ 1110line 1109 didn't jump to line 1110, because the loop on line 1109 never started

1110 if imp in sys.modules and sys.modules[imp] is not None: 

1111 outfile.write(u"import {}\n".format(imp)) 

1112 self._save(outfile) 

1113 finally: 

1114 self._rename(tmp) 

1115 

1116 def freeze(self): 

1117 """Make this config, and all subconfigs, read-only. 

1118 """ 

1119 self._frozen = True 

1120 for field in self._fields.values(): 

1121 field.freeze(self) 

1122 

1123 def _save(self, outfile): 

1124 """Save this config to an open stream object. 

1125 

1126 Parameters 

1127 ---------- 

1128 outfile : file-like object 

1129 Destination file object write the config into. Accepts strings not 

1130 bytes. 

1131 """ 

1132 for field in self._fields.values(): 

1133 field.save(outfile, self) 

1134 

1135 def _collectImports(self): 

1136 """Adds module containing self to the list of things to import and 

1137 then loops over all the fields in the config calling a corresponding 

1138 collect method. The field method will call _collectImports on any 

1139 configs it may own and return the set of things to import. This 

1140 returned set will be merged with the set of imports for this config 

1141 class. 

1142 """ 

1143 self._imports.add(self.__module__) 

1144 for name, field in self._fields.items(): 

1145 field._collectImports(self, self._imports) 

1146 

1147 def toDict(self): 

1148 """Make a dictionary of field names and their values. 

1149 

1150 Returns 

1151 ------- 

1152 dict_ : `dict` 

1153 Dictionary with keys that are `~lsst.pex.config.Field` names. 

1154 Values are `~lsst.pex.config.Field` values. 

1155 

1156 See also 

1157 -------- 

1158 lsst.pex.config.Field.toDict 

1159 

1160 Notes 

1161 ----- 

1162 This method uses the `~lsst.pex.config.Field.toDict` method of 

1163 individual fields. Subclasses of `~lsst.pex.config.Field` may need to 

1164 implement a ``toDict`` method for *this* method to work. 

1165 """ 

1166 dict_ = {} 

1167 for name, field in self._fields.items(): 

1168 dict_[name] = field.toDict(self) 

1169 return dict_ 

1170 

1171 def names(self): 

1172 """Get all the field names in the config, recursively. 

1173 

1174 Returns 

1175 ------- 

1176 names : `list` of `str` 

1177 Field names. 

1178 """ 

1179 # 

1180 # Rather than sort out the recursion all over again use the 

1181 # pre-existing saveToStream() 

1182 # 

1183 with io.StringIO() as strFd: 

1184 self.saveToStream(strFd, "config") 

1185 contents = strFd.getvalue() 

1186 strFd.close() 

1187 # 

1188 # Pull the names out of the dumped config 

1189 # 

1190 keys = [] 

1191 for line in contents.split("\n"): 

1192 if re.search(r"^((assert|import)\s+|\s*$|#)", line): 

1193 continue 

1194 

1195 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line) 

1196 if mat: 

1197 keys.append(mat.group(1)) 

1198 

1199 return keys 

1200 

1201 def _rename(self, name): 

1202 """Rename this config object in its parent `~lsst.pex.config.Config`. 

1203 

1204 Parameters 

1205 ---------- 

1206 name : `str` 

1207 New name for this config in its parent `~lsst.pex.config.Config`. 

1208 

1209 Notes 

1210 ----- 

1211 This method uses the `~lsst.pex.config.Field.rename` method of 

1212 individual `lsst.pex.config.Field` instances. 

1213 `lsst.pex.config.Field` subclasses may need to implement a ``rename`` 

1214 method for *this* method to work. 

1215 

1216 See also 

1217 -------- 

1218 lsst.pex.config.Field.rename 

1219 """ 

1220 self._name = name 

1221 for field in self._fields.values(): 

1222 field.rename(self) 

1223 

1224 def validate(self): 

1225 """Validate the Config, raising an exception if invalid. 

1226 

1227 Raises 

1228 ------ 

1229 lsst.pex.config.FieldValidationError 

1230 Raised if verification fails. 

1231 

1232 Notes 

1233 ----- 

1234 The base class implementation performs type checks on all fields by 

1235 calling their `~lsst.pex.config.Field.validate` methods. 

1236 

1237 Complex single-field validation can be defined by deriving new Field 

1238 types. For convenience, some derived `lsst.pex.config.Field`-types 

1239 (`~lsst.pex.config.ConfigField` and 

1240 `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config` 

1241 that handle recursing into subconfigs. 

1242 

1243 Inter-field relationships should only be checked in derived 

1244 `~lsst.pex.config.Config` classes after calling this method, and base 

1245 validation is complete. 

1246 """ 

1247 for field in self._fields.values(): 

1248 field.validate(self) 

1249 

1250 def formatHistory(self, name, **kwargs): 

1251 """Format a configuration field's history to a human-readable format. 

1252 

1253 Parameters 

1254 ---------- 

1255 name : `str` 

1256 Name of a `~lsst.pex.config.Field` in this config. 

1257 kwargs 

1258 Keyword arguments passed to `lsst.pex.config.history.format`. 

1259 

1260 Returns 

1261 ------- 

1262 history : `str` 

1263 A string containing the formatted history. 

1264 

1265 See also 

1266 -------- 

1267 lsst.pex.config.history.format 

1268 """ 

1269 import lsst.pex.config.history as pexHist 

1270 return pexHist.format(self, name, **kwargs) 

1271 

1272 history = property(lambda x: x._history) 1272 ↛ exitline 1272 didn't run the lambda on line 1272

1273 """Read-only history. 

1274 """ 

1275 

1276 def __setattr__(self, attr, value, at=None, label="assignment"): 

1277 """Set an attribute (such as a field's value). 

1278 

1279 Notes 

1280 ----- 

1281 Unlike normal Python objects, `~lsst.pex.config.Config` objects are 

1282 locked such that no additional attributes nor properties may be added 

1283 to them dynamically. 

1284 

1285 Although this is not the standard Python behavior, it helps to protect 

1286 users from accidentally mispelling a field name, or trying to set a 

1287 non-existent field. 

1288 """ 

1289 if attr in self._fields: 

1290 if self._fields[attr].deprecated is not None: 1290 ↛ 1291line 1290 didn't jump to line 1291, because the condition on line 1290 was never true

1291 fullname = _joinNamePath(self._name, self._fields[attr].name) 

1292 warnings.warn(f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}", 

1293 FutureWarning, stacklevel=2) 

1294 if at is None: 1294 ↛ 1297line 1294 didn't jump to line 1297, because the condition on line 1294 was never false

1295 at = getCallStack() 

1296 # This allows Field descriptors to work. 

1297 self._fields[attr].__set__(self, value, at=at, label=label) 

1298 elif hasattr(getattr(self.__class__, attr, None), '__set__'): 1298 ↛ 1300line 1298 didn't jump to line 1300, because the condition on line 1298 was never true

1299 # This allows properties and other non-Field descriptors to work. 

1300 return object.__setattr__(self, attr, value) 

1301 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"): 1301 ↛ 1306line 1301 didn't jump to line 1306, because the condition on line 1301 was never false

1302 # This allows specific private attributes to work. 

1303 self.__dict__[attr] = value 

1304 else: 

1305 # We throw everything else. 

1306 raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr)) 

1307 

1308 def __delattr__(self, attr, at=None, label="deletion"): 

1309 if attr in self._fields: 

1310 if at is None: 

1311 at = getCallStack() 

1312 self._fields[attr].__delete__(self, at=at, label=label) 

1313 else: 

1314 object.__delattr__(self, attr) 

1315 

1316 def __eq__(self, other): 

1317 if type(other) == type(self): 1317 ↛ 1318line 1317 didn't jump to line 1318, because the condition on line 1317 was never true

1318 for name in self._fields: 

1319 thisValue = getattr(self, name) 

1320 otherValue = getattr(other, name) 

1321 if isinstance(thisValue, float) and math.isnan(thisValue): 

1322 if not math.isnan(otherValue): 

1323 return False 

1324 elif thisValue != otherValue: 

1325 return False 

1326 return True 

1327 return False 

1328 

1329 def __ne__(self, other): 

1330 return not self.__eq__(other) 

1331 

1332 def __str__(self): 

1333 return str(self.toDict()) 

1334 

1335 def __repr__(self): 

1336 return "%s(%s)" % ( 

1337 _typeStr(self), 

1338 ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None) 

1339 ) 

1340 

1341 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None): 

1342 """Compare this configuration to another `~lsst.pex.config.Config` for 

1343 equality. 

1344 

1345 Parameters 

1346 ---------- 

1347 other : `lsst.pex.config.Config` 

1348 Other `~lsst.pex.config.Config` object to compare against this 

1349 config. 

1350 shortcut : `bool`, optional 

1351 If `True`, return as soon as an inequality is found. Default is 

1352 `True`. 

1353 rtol : `float`, optional 

1354 Relative tolerance for floating point comparisons. 

1355 atol : `float`, optional 

1356 Absolute tolerance for floating point comparisons. 

1357 output : callable, optional 

1358 A callable that takes a string, used (possibly repeatedly) to 

1359 report inequalities. 

1360 

1361 Returns 

1362 ------- 

1363 isEqual : `bool` 

1364 `True` when the two `lsst.pex.config.Config` instances are equal. 

1365 `False` if there is an inequality. 

1366 

1367 See also 

1368 -------- 

1369 lsst.pex.config.compareConfigs 

1370 

1371 Notes 

1372 ----- 

1373 Unselected targets of `~lsst.pex.config.RegistryField` fields and 

1374 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields 

1375 are not considered by this method. 

1376 

1377 Floating point comparisons are performed by `numpy.allclose`. 

1378 """ 

1379 name1 = self._name if self._name is not None else "config" 

1380 name2 = other._name if other._name is not None else "config" 

1381 name = getComparisonName(name1, name2) 

1382 return compareConfigs(name, self, other, shortcut=shortcut, 

1383 rtol=rtol, atol=atol, output=output) 

1384 

1385 

1386def unreduceConfig(cls, stream): 

1387 """Create a `~lsst.pex.config.Config` from a stream. 

1388 

1389 Parameters 

1390 ---------- 

1391 cls : `lsst.pex.config.Config`-type 

1392 A `lsst.pex.config.Config` type (not an instance) that is instantiated 

1393 with configurations in the ``stream``. 

1394 stream : file-like object, `str`, or compiled string 

1395 Stream containing configuration override code. 

1396 

1397 Returns 

1398 ------- 

1399 config : `lsst.pex.config.Config` 

1400 Config instance. 

1401 

1402 See also 

1403 -------- 

1404 lsst.pex.config.Config.loadFromStream 

1405 """ 

1406 config = cls() 

1407 config.loadFromStream(stream) 

1408 return config