Coverage for python/lsst/pipe/tasks/configurableActions/_configurableActionStructField.py: 25%

183 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-26 03:18 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ("ConfigurableActionStructField", "ConfigurableActionStruct") 

24 

25from types import SimpleNamespace 

26from typing import ( 

27 Iterable, 

28 Mapping, 

29 Optional, 

30 TypeVar, 

31 Union, 

32 Type, 

33 Tuple, 

34 List, 

35 Any, 

36 Dict, 

37 Iterator, 

38 Generic, 

39 overload 

40) 

41from types import GenericAlias 

42 

43from lsst.pex.config.config import Config, Field, FieldValidationError, _typeStr, _joinNamePath 

44from lsst.pex.config.comparison import compareConfigs, compareScalars, getComparisonName 

45from lsst.pex.config.callStack import StackFrame, getCallStack, getStackFrame 

46from lsst.pipe.base import Struct 

47 

48from . import ConfigurableAction, ActionTypeVar 

49 

50import weakref 

51 

52 

53class ConfigurableActionStructUpdater: 

54 """This descriptor exists to abstract the logic of using a dictionary to 

55 update a ConfigurableActionStruct through attribute assignment. This is 

56 useful in the context of setting configuration through pipelines or on 

57 the command line. 

58 """ 

59 def __set__(self, instance: ConfigurableActionStruct, 

60 value: Union[Mapping[str, ConfigurableAction], ConfigurableActionStruct]) -> None: 

61 if isinstance(value, Mapping): 

62 pass 

63 elif isinstance(value, ConfigurableActionStruct): 

64 # If the update target is a ConfigurableActionStruct, get the 

65 # internal dictionary 

66 value = value._attrs 

67 else: 

68 raise ValueError("Can only update a ConfigurableActionStruct with an instance of such, or a " 

69 "mapping") 

70 for name, action in value.items(): 

71 setattr(instance, name, action) 

72 

73 def __get__(self, instance, objtype=None) -> None: 

74 # This descriptor does not support fetching any value 

75 return None 

76 

77 

78class ConfigurableActionStructRemover: 

79 """This descriptor exists to abstract the logic of removing an interable 

80 of action names from a ConfigurableActionStruct at one time using 

81 attribute assignment. This is useful in the context of setting 

82 configuration through pipelines or on the command line. 

83 

84 Raises 

85 ------ 

86 AttributeError 

87 Raised if an attribute specified for removal does not exist in the 

88 ConfigurableActionStruct 

89 """ 

90 def __set__(self, instance: ConfigurableActionStruct, 

91 value: Union[str, Iterable[str]]) -> None: 

92 # strings are iterable, but not in the way that is intended. If a 

93 # single name is specified, turn it into a tuple before attempting 

94 # to remove the attribute 

95 if isinstance(value, str): 

96 value = (value, ) 

97 for name in value: 

98 delattr(instance, name) 

99 

100 def __get__(self, instance, objtype=None) -> None: 

101 # This descriptor does not support fetching any value 

102 return None 

103 

104 

105class ConfigurableActionStruct(Generic[ActionTypeVar]): 

106 """A ConfigurableActionStruct is the storage backend class that supports 

107 the ConfigurableActionStructField. This class should not be created 

108 directly. 

109 

110 This class allows managing a collection of `ConfigurableActions` with a 

111 struct like interface, that is to say in an attribute like notation. 

112 

113 Attributes can be dynamically added or removed as such: 

114 

115 ConfigurableActionStructInstance.variable1 = a_configurable_action 

116 del ConfigurableActionStructInstance.variable1 

117 

118 Each action is then available to be individually configured as a normal 

119 `lsst.pex.config.Config` object. 

120 

121 ConfigurableActionStruct supports two special convenance attributes. 

122 

123 The first is `update`. You may assign a dict of `ConfigurableActions` or 

124 a `ConfigurableActionStruct` to this attribute which will update the 

125 `ConfigurableActionStruct` on which the attribute is invoked such that it 

126 will be updated to contain the entries specified by the structure on the 

127 right hand side of the equals sign. 

128 

129 The second convenience attribute is named remove. You may assign an 

130 iterable of strings which correspond to attribute names on the 

131 `ConfigurableActionStruct`. All of the corresponding attributes will then 

132 be removed. If any attribute does not exist, an `AttributeError` will be 

133 raised. Any attributes in the Iterable prior to the name which raises will 

134 have been removed from the `ConfigurableActionStruct` 

135 """ 

136 # declare attributes that are set with __setattr__ 

137 _config_: weakref.ref 

138 _attrs: Dict[str, ActionTypeVar] 

139 _field: ConfigurableActionStructField 

140 _history: List[tuple] 

141 

142 # create descriptors to handle special update and remove behavior 

143 update = ConfigurableActionStructUpdater() 

144 remove = ConfigurableActionStructRemover() 

145 

146 def __init__(self, config: Config, field: ConfigurableActionStructField, 

147 value: Mapping[str, ConfigurableAction], at: Any, label: str): 

148 object.__setattr__(self, '_config_', weakref.ref(config)) 

149 object.__setattr__(self, '_attrs', {}) 

150 object.__setattr__(self, '_field', field) 

151 object.__setattr__(self, '_history', []) 

152 

153 self.history.append(("Struct initialized", at, label)) 

154 

155 if value is not None: 

156 for k, v in value.items(): 

157 setattr(self, k, v) 

158 

159 @property 

160 def _config(self) -> Config: 

161 # Config Fields should never outlive their config class instance 

162 # assert that as such here 

163 value = self._config_() 

164 assert(value is not None) 

165 return value 

166 

167 @property 

168 def history(self) -> List[tuple]: 

169 return self._history 

170 

171 @property 

172 def fieldNames(self) -> Iterable[str]: 

173 return self._attrs.keys() 

174 

175 def __setattr__(self, attr: str, value: Union[ActionTypeVar, Type[ActionTypeVar]], 

176 at=None, label='setattr', setHistory=False) -> None: 

177 

178 if hasattr(self._config, '_frozen') and self._config._frozen: 

179 msg = "Cannot modify a frozen Config. "\ 

180 f"Attempting to set item {attr} to value {value}" 

181 raise FieldValidationError(self._field, self._config, msg) 

182 

183 # verify that someone has not passed a string with a space or leading 

184 # number or something through the dict assignment update interface 

185 if not attr.isidentifier(): 

186 raise ValueError("Names used in ConfigurableStructs must be valid as python variable names") 

187 

188 if attr not in (self.__dict__.keys() | type(self).__dict__.keys()): 

189 base_name = _joinNamePath(self._config._name, self._field.name) 

190 name = _joinNamePath(base_name, attr) 

191 if at is None: 

192 at = getCallStack() 

193 if isinstance(value, ConfigurableAction): 

194 valueInst = type(value)(__name=name, __at=at, __label=label, **value._storage) 

195 else: 

196 valueInst = value(__name=name, __at=at, __label=label) 

197 self._attrs[attr] = valueInst 

198 else: 

199 super().__setattr__(attr, value) 

200 

201 def __getattr__(self, attr) -> Any: 

202 if attr in object.__getattribute__(self, '_attrs'): 

203 result = self._attrs[attr] 

204 result.identity = attr 

205 return result 

206 else: 

207 super().__getattribute__(attr) 

208 

209 def __delattr__(self, name): 

210 if name in self._attrs: 

211 del self._attrs[name] 

212 else: 

213 super().__delattr__(name) 

214 

215 def __iter__(self) -> Iterator[ActionTypeVar]: 

216 for name in self.fieldNames: 

217 yield getattr(self, name) 

218 

219 def items(self) -> Iterable[Tuple[str, ActionTypeVar]]: 

220 for name in self.fieldNames: 

221 yield name, getattr(self, name) 

222 

223 def __bool__(self) -> bool: 

224 return bool(self._attrs) 

225 

226 

227T = TypeVar("T", bound="ConfigurableActionStructField") 

228 

229 

230class ConfigurableActionStructField(Field[ActionTypeVar]): 

231 r"""`ConfigurableActionStructField` is a `~lsst.pex.config.Field` subclass 

232 that allows `ConfigurableAction`\ s to be organized in a 

233 `~lsst.pex.config.Config` class in a manner similar to how a 

234 `~lsst.pipe.base.Struct` works. 

235 

236 This class implements a `ConfigurableActionStruct` as an intermediary 

237 object to organize the `ConfigurableActions`. See it's documentation for 

238 futher information. 

239 """ 

240 # specify StructClass to make this more generic for potential future 

241 # inheritance 

242 StructClass = ConfigurableActionStruct 

243 

244 # Explicitly annotate these on the class, they are present in the base 

245 # class through injection, so type systems have trouble seeing them. 

246 name: str 

247 default: Optional[Mapping[str, ConfigurableAction]] 

248 

249 def __init__(self, doc: str, default: Optional[Mapping[str, ConfigurableAction]] = None, 

250 optional: bool = False, 

251 deprecated=None): 

252 source = getStackFrame() 

253 self._setup(doc=doc, dtype=self.__class__, default=default, check=None, 

254 optional=optional, source=source, deprecated=deprecated) 

255 

256 def __class_getitem__(cls, params): 

257 return GenericAlias(cls, params) 

258 

259 def __set__(self, instance: Config, 

260 value: Union[None, Mapping[str, ConfigurableAction], 

261 SimpleNamespace, 

262 Struct, 

263 ConfigurableActionStruct, 

264 ConfigurableActionStructField, 

265 Type[ConfigurableActionStructField]], 

266 at: Iterable[StackFrame] = None, label: str = 'assigment'): 

267 if instance._frozen: 

268 msg = "Cannot modify a frozen Config. "\ 

269 "Attempting to set field to value %s" % value 

270 raise FieldValidationError(self, instance, msg) 

271 

272 if at is None: 

273 at = getCallStack() 

274 

275 if value is None or (self.default is not None and self.default == value): 

276 value = self.StructClass(instance, self, value, at=at, label=label) 

277 else: 

278 # An actual value is being assigned check for what it is 

279 if isinstance(value, self.StructClass): 

280 # If this is a ConfigurableActionStruct, we need to make our 

281 # own copy that references this current field 

282 value = self.StructClass(instance, self, value._attrs, at=at, label=label) 

283 elif isinstance(value, (SimpleNamespace, Struct)): 

284 # If this is a a python analogous container, we need to make 

285 # a ConfigurableActionStruct initialized with this data 

286 value = self.StructClass(instance, self, vars(value), at=at, label=label) 

287 

288 elif type(value) == ConfigurableActionStructField: 

289 raise ValueError("ConfigurableActionStructFields can only be used in a class body declaration" 

290 f"Use a {self.StructClass}, SimpleNamespace or Struct") 

291 else: 

292 raise ValueError(f"Unrecognized value {value}, cannot be assigned to this field") 

293 

294 history = instance._history.setdefault(self.name, []) 

295 history.append((value, at, label)) 

296 

297 if not isinstance(value, ConfigurableActionStruct): 

298 raise FieldValidationError(self, instance, 

299 "Can only assign things that are subclasses of Configurable Action") 

300 instance._storage[self.name] = value 

301 

302 @overload 

303 def __get__( 

304 self, 

305 instance: None, 

306 owner: Any = None, 

307 at: Any = None, 

308 label: str = 'default' 

309 ) -> ConfigurableActionStruct[ActionTypeVar]: 

310 ... 

311 

312 @overload 

313 def __get__( 

314 self, 

315 instance: Config, 

316 owner: Any = None, 

317 at: Any = None, 

318 label: str = 'default' 

319 ) -> ConfigurableActionStruct[ActionTypeVar]: 

320 ... 

321 

322 def __get__( 

323 self, 

324 instance, 

325 owner=None, 

326 at=None, 

327 label='default' 

328 ): 

329 if instance is None or not isinstance(instance, Config): 

330 return self 

331 else: 

332 field: Optional[ConfigurableActionStruct] = instance._storage[self.name] 

333 return field 

334 

335 def rename(self, instance: Config): 

336 actionStruct: ConfigurableActionStruct = self.__get__(instance) 

337 if actionStruct is not None: 

338 for k, v in actionStruct.items(): 

339 base_name = _joinNamePath(instance._name, self.name) 

340 fullname = _joinNamePath(base_name, k) 

341 v._rename(fullname) 

342 

343 def validate(self, instance: Config): 

344 value = self.__get__(instance) 

345 if value is not None: 

346 for item in value: 

347 item.validate() 

348 

349 def toDict(self, instance): 

350 actionStruct = self.__get__(instance) 

351 if actionStruct is None: 

352 return None 

353 

354 dict_ = {k: v.toDict() for k, v in actionStruct.items()} 

355 

356 return dict_ 

357 

358 def save(self, outfile, instance): 

359 actionStruct = self.__get__(instance) 

360 fullname = _joinNamePath(instance._name, self.name) 

361 

362 # Ensure that a struct is always empty before assigning to it. 

363 outfile.write(f"{fullname}=None\n") 

364 

365 if actionStruct is None: 

366 return 

367 

368 for _, v in sorted(actionStruct.items()): 

369 outfile.write(u"{}={}()\n".format(v._name, _typeStr(v))) 

370 v._save(outfile) 

371 

372 def freeze(self, instance): 

373 actionStruct = self.__get__(instance) 

374 if actionStruct is not None: 

375 for v in actionStruct: 

376 v.freeze() 

377 

378 def _collectImports(self, instance, imports): 

379 # docstring inherited from Field 

380 actionStruct = self.__get__(instance) 

381 for v in actionStruct: 

382 v._collectImports() 

383 imports |= v._imports 

384 

385 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

386 """Compare two fields for equality. 

387 

388 Parameters 

389 ---------- 

390 instance1 : `lsst.pex.config.Config` 

391 Left-hand side config instance to compare. 

392 instance2 : `lsst.pex.config.Config` 

393 Right-hand side config instance to compare. 

394 shortcut : `bool` 

395 If `True`, this function returns as soon as an inequality if found. 

396 rtol : `float` 

397 Relative tolerance for floating point comparisons. 

398 atol : `float` 

399 Absolute tolerance for floating point comparisons. 

400 output : callable 

401 A callable that takes a string, used (possibly repeatedly) to 

402 report inequalities. 

403 

404 Returns 

405 ------- 

406 isEqual : bool 

407 `True` if the fields are equal, `False` otherwise. 

408 

409 Notes 

410 ----- 

411 Floating point comparisons are performed by `numpy.allclose`. 

412 """ 

413 d1: ConfigurableActionStruct = getattr(instance1, self.name) 

414 d2: ConfigurableActionStruct = getattr(instance2, self.name) 

415 name = getComparisonName( 

416 _joinNamePath(instance1._name, self.name), 

417 _joinNamePath(instance2._name, self.name) 

418 ) 

419 if not compareScalars(f"keys for {name}", set(d1.fieldNames), set(d2.fieldNames), output=output): 

420 return False 

421 equal = True 

422 for k, v1 in d1.items(): 

423 v2 = getattr(d2, k) 

424 result = compareConfigs(f"{name}.{k}", v1, v2, shortcut=shortcut, 

425 rtol=rtol, atol=atol, output=output) 

426 if not result and shortcut: 

427 return False 

428 equal = equal and result 

429 return equal