Coverage for python/lsst/pipe/tasks/configurableActions/_configurableActionStructField.py: 24%

181 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-07-06 01:42 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ("ConfigurableActionStructField", "ConfigurableActionStruct") 

24 

25from types import SimpleNamespace 

26from typing import ( 

27 Iterable, 

28 Mapping, 

29 Optional, 

30 TypeVar, 

31 Union, 

32 Type, 

33 Tuple, 

34 List, 

35 Any, 

36 Dict, 

37 Iterator, 

38 Generic, 

39 overload 

40) 

41from types import GenericAlias 

42 

43from lsst.pex.config.config import Config, Field, FieldValidationError, _typeStr, _joinNamePath 

44from lsst.pex.config.comparison import compareConfigs, compareScalars, getComparisonName 

45from lsst.pex.config.callStack import StackFrame, getCallStack, getStackFrame 

46from lsst.pipe.base import Struct 

47 

48from . import ConfigurableAction, ActionTypeVar 

49 

50import weakref 

51 

52 

53class ConfigurableActionStructUpdater: 

54 """This descriptor exists to abstract the logic of using a dictionary to 

55 update a ConfigurableActionStruct through attribute assignment. This is 

56 useful in the context of setting configuration through pipelines or on 

57 the command line. 

58 """ 

59 def __set__(self, instance: ConfigurableActionStruct, 

60 value: Union[Mapping[str, ConfigurableAction], ConfigurableActionStruct]) -> None: 

61 if isinstance(value, Mapping): 

62 pass 

63 elif isinstance(value, ConfigurableActionStruct): 

64 # If the update target is a ConfigurableActionStruct, get the 

65 # internal dictionary 

66 value = value._attrs 

67 else: 

68 raise ValueError("Can only update a ConfigurableActionStruct with an instance of such, or a " 

69 "mapping") 

70 for name, action in value.items(): 

71 setattr(instance, name, action) 

72 

73 def __get__(self, instance, objtype=None) -> None: 

74 # This descriptor does not support fetching any value 

75 return None 

76 

77 

78class ConfigurableActionStructRemover: 

79 """This descriptor exists to abstract the logic of removing an interable 

80 of action names from a ConfigurableActionStruct at one time using 

81 attribute assignment. This is useful in the context of setting 

82 configuration through pipelines or on the command line. 

83 

84 Raises 

85 ------ 

86 AttributeError 

87 Raised if an attribute specified for removal does not exist in the 

88 ConfigurableActionStruct 

89 """ 

90 def __set__(self, instance: ConfigurableActionStruct, 

91 value: Union[str, Iterable[str]]) -> None: 

92 # strings are iterable, but not in the way that is intended. If a 

93 # single name is specified, turn it into a tuple before attempting 

94 # to remove the attribute 

95 if isinstance(value, str): 

96 value = (value, ) 

97 for name in value: 

98 delattr(instance, name) 

99 

100 def __get__(self, instance, objtype=None) -> None: 

101 # This descriptor does not support fetching any value 

102 return None 

103 

104 

105class ConfigurableActionStruct(Generic[ActionTypeVar]): 

106 """A ConfigurableActionStruct is the storage backend class that supports 

107 the ConfigurableActionStructField. This class should not be created 

108 directly. 

109 

110 This class allows managing a collection of `ConfigurableActions` with a 

111 struct like interface, that is to say in an attribute like notation. 

112 

113 Attributes can be dynamically added or removed as such: 

114 

115 ConfigurableActionStructInstance.variable1 = a_configurable_action 

116 del ConfigurableActionStructInstance.variable1 

117 

118 Each action is then available to be individually configured as a normal 

119 `lsst.pex.config.Config` object. 

120 

121 ConfigurableActionStruct supports two special convenance attributes. 

122 

123 The first is `update`. You may assign a dict of `ConfigurableActions` or 

124 a `ConfigurableActionStruct` to this attribute which will update the 

125 `ConfigurableActionStruct` on which the attribute is invoked such that it 

126 will be updated to contain the entries specified by the structure on the 

127 right hand side of the equals sign. 

128 

129 The second convenience attribute is named remove. You may assign an 

130 iterable of strings which correspond to attribute names on the 

131 `ConfigurableActionStruct`. All of the corresponding attributes will then 

132 be removed. If any attribute does not exist, an `AttributeError` will be 

133 raised. Any attributes in the Iterable prior to the name which raises will 

134 have been removed from the `ConfigurableActionStruct` 

135 """ 

136 # declare attributes that are set with __setattr__ 

137 _config_: weakref.ref 

138 _attrs: Dict[str, ActionTypeVar] 

139 _field: ConfigurableActionStructField 

140 _history: List[tuple] 

141 

142 # create descriptors to handle special update and remove behavior 

143 update = ConfigurableActionStructUpdater() 

144 remove = ConfigurableActionStructRemover() 

145 

146 def __init__(self, config: Config, field: ConfigurableActionStructField, 

147 value: Mapping[str, ConfigurableAction], at: Any, label: str): 

148 object.__setattr__(self, '_config_', weakref.ref(config)) 

149 object.__setattr__(self, '_attrs', {}) 

150 object.__setattr__(self, '_field', field) 

151 object.__setattr__(self, '_history', []) 

152 

153 self.history.append(("Struct initialized", at, label)) 

154 

155 if value is not None: 

156 for k, v in value.items(): 

157 setattr(self, k, v) 

158 

159 @property 

160 def _config(self) -> Config: 

161 # Config Fields should never outlive their config class instance 

162 # assert that as such here 

163 value = self._config_() 

164 assert(value is not None) 

165 return value 

166 

167 @property 

168 def history(self) -> List[tuple]: 

169 return self._history 

170 

171 @property 

172 def fieldNames(self) -> Iterable[str]: 

173 return self._attrs.keys() 

174 

175 def __setattr__(self, attr: str, value: Union[ActionTypeVar, Type[ActionTypeVar]], 

176 at=None, label='setattr', setHistory=False) -> None: 

177 

178 if hasattr(self._config, '_frozen') and self._config._frozen: 

179 msg = "Cannot modify a frozen Config. "\ 

180 f"Attempting to set item {attr} to value {value}" 

181 raise FieldValidationError(self._field, self._config, msg) 

182 

183 # verify that someone has not passed a string with a space or leading 

184 # number or something through the dict assignment update interface 

185 if not attr.isidentifier(): 

186 raise ValueError("Names used in ConfigurableStructs must be valid as python variable names") 

187 

188 if attr not in (self.__dict__.keys() | type(self).__dict__.keys()): 

189 base_name = _joinNamePath(self._config._name, self._field.name) 

190 name = _joinNamePath(base_name, attr) 

191 if at is None: 

192 at = getCallStack() 

193 if isinstance(value, ConfigurableAction): 

194 valueInst = type(value)(__name=name, __at=at, __label=label, **value._storage) 

195 else: 

196 valueInst = value(__name=name, __at=at, __label=label) 

197 self._attrs[attr] = valueInst 

198 else: 

199 super().__setattr__(attr, value) 

200 

201 def __getattr__(self, attr) -> Any: 

202 if attr in object.__getattribute__(self, '_attrs'): 

203 result = self._attrs[attr] 

204 result.identity = attr 

205 return result 

206 else: 

207 super().__getattribute__(attr) 

208 

209 def __delattr__(self, name): 

210 if name in self._attrs: 

211 del self._attrs[name] 

212 else: 

213 super().__delattr__(name) 

214 

215 def __iter__(self) -> Iterator[ActionTypeVar]: 

216 for name in self.fieldNames: 

217 yield getattr(self, name) 

218 

219 def items(self) -> Iterable[Tuple[str, ActionTypeVar]]: 

220 for name in self.fieldNames: 

221 yield name, getattr(self, name) 

222 

223 

224T = TypeVar("T", bound="ConfigurableActionStructField") 

225 

226 

227class ConfigurableActionStructField(Field[ActionTypeVar]): 

228 r"""`ConfigurableActionStructField` is a `~lsst.pex.config.Field` subclass 

229 that allows `ConfigurableAction`\ s to be organized in a 

230 `~lsst.pex.config.Config` class in a manner similar to how a 

231 `~lsst.pipe.base.Struct` works. 

232 

233 This class implements a `ConfigurableActionStruct` as an intermediary 

234 object to organize the `ConfigurableActions`. See it's documentation for 

235 futher information. 

236 """ 

237 # specify StructClass to make this more generic for potential future 

238 # inheritance 

239 StructClass = ConfigurableActionStruct 

240 

241 # Explicitly annotate these on the class, they are present in the base 

242 # class through injection, so type systems have trouble seeing them. 

243 name: str 

244 default: Optional[Mapping[str, ConfigurableAction]] 

245 

246 def __init__(self, doc: str, default: Optional[Mapping[str, ConfigurableAction]] = None, 

247 optional: bool = False, 

248 deprecated=None): 

249 source = getStackFrame() 

250 self._setup(doc=doc, dtype=self.__class__, default=default, check=None, 

251 optional=optional, source=source, deprecated=deprecated) 

252 

253 def __class_getitem__(cls, params): 

254 return GenericAlias(cls, params) 

255 

256 def __set__(self, instance: Config, 

257 value: Union[None, Mapping[str, ConfigurableAction], 

258 SimpleNamespace, 

259 Struct, 

260 ConfigurableActionStruct, 

261 ConfigurableActionStructField, 

262 Type[ConfigurableActionStructField]], 

263 at: Iterable[StackFrame] = None, label: str = 'assigment'): 

264 if instance._frozen: 

265 msg = "Cannot modify a frozen Config. "\ 

266 "Attempting to set field to value %s" % value 

267 raise FieldValidationError(self, instance, msg) 

268 

269 if at is None: 

270 at = getCallStack() 

271 

272 if value is None or (self.default is not None and self.default == value): 

273 value = self.StructClass(instance, self, value, at=at, label=label) 

274 else: 

275 # An actual value is being assigned check for what it is 

276 if isinstance(value, self.StructClass): 

277 # If this is a ConfigurableActionStruct, we need to make our own 

278 # copy that references this current field 

279 value = self.StructClass(instance, self, value._attrs, at=at, label=label) 

280 elif isinstance(value, (SimpleNamespace, Struct)): 

281 # If this is a a python analogous container, we need to make 

282 # a ConfigurableActionStruct initialized with this data 

283 value = self.StructClass(instance, self, vars(value), at=at, label=label) 

284 

285 elif type(value) == ConfigurableActionStructField: 

286 raise ValueError("ConfigurableActionStructFields can only be used in a class body declaration" 

287 f"Use a {self.StructClass}, SimpleNamespace or Struct") 

288 else: 

289 raise ValueError(f"Unrecognized value {value}, cannot be assigned to this field") 

290 

291 history = instance._history.setdefault(self.name, []) 

292 history.append((value, at, label)) 

293 

294 if not isinstance(value, ConfigurableActionStruct): 

295 raise FieldValidationError(self, instance, 

296 "Can only assign things that are subclasses of Configurable Action") 

297 instance._storage[self.name] = value 

298 

299 @overload 

300 def __get__( 

301 self, 

302 instance: None, 

303 owner: Any = None, 

304 at: Any = None, 

305 label: str = 'default' 

306 ) -> ConfigurableActionStruct[ActionTypeVar]: 

307 ... 

308 

309 @overload 

310 def __get__( 

311 self, 

312 instance: Config, 

313 owner: Any = None, 

314 at: Any = None, 

315 label: str = 'default' 

316 ) -> ConfigurableActionStruct[ActionTypeVar]: 

317 ... 

318 

319 def __get__( 

320 self, 

321 instance, 

322 owner=None, 

323 at=None, 

324 label='default' 

325 ): 

326 if instance is None or not isinstance(instance, Config): 

327 return self 

328 else: 

329 field: Optional[ConfigurableActionStruct] = instance._storage[self.name] 

330 return field 

331 

332 def rename(self, instance: Config): 

333 actionStruct: ConfigurableActionStruct = self.__get__(instance) 

334 if actionStruct is not None: 

335 for k, v in actionStruct.items(): 

336 base_name = _joinNamePath(instance._name, self.name) 

337 fullname = _joinNamePath(base_name, k) 

338 v._rename(fullname) 

339 

340 def validate(self, instance: Config): 

341 value = self.__get__(instance) 

342 if value is not None: 

343 for item in value: 

344 item.validate() 

345 

346 def toDict(self, instance): 

347 actionStruct = self.__get__(instance) 

348 if actionStruct is None: 

349 return None 

350 

351 dict_ = {k: v.toDict() for k, v in actionStruct.items()} 

352 

353 return dict_ 

354 

355 def save(self, outfile, instance): 

356 actionStruct = self.__get__(instance) 

357 fullname = _joinNamePath(instance._name, self.name) 

358 if actionStruct is None: 

359 outfile.write(u"{}={!r}\n".format(fullname, actionStruct)) 

360 return 

361 

362 for v in actionStruct: 

363 outfile.write(u"{}={}()\n".format(v._name, _typeStr(v))) 

364 v._save(outfile) 

365 

366 def freeze(self, instance): 

367 actionStruct = self.__get__(instance) 

368 if actionStruct is not None: 

369 for v in actionStruct: 

370 v.freeze() 

371 

372 def _collectImports(self, instance, imports): 

373 # docstring inherited from Field 

374 actionStruct = self.__get__(instance) 

375 for v in actionStruct: 

376 v._collectImports() 

377 imports |= v._imports 

378 

379 def _compare(self, instance1, instance2, shortcut, rtol, atol, output): 

380 """Compare two fields for equality. 

381 

382 Parameters 

383 ---------- 

384 instance1 : `lsst.pex.config.Config` 

385 Left-hand side config instance to compare. 

386 instance2 : `lsst.pex.config.Config` 

387 Right-hand side config instance to compare. 

388 shortcut : `bool` 

389 If `True`, this function returns as soon as an inequality if found. 

390 rtol : `float` 

391 Relative tolerance for floating point comparisons. 

392 atol : `float` 

393 Absolute tolerance for floating point comparisons. 

394 output : callable 

395 A callable that takes a string, used (possibly repeatedly) to 

396 report inequalities. 

397 

398 Returns 

399 ------- 

400 isEqual : bool 

401 `True` if the fields are equal, `False` otherwise. 

402 

403 Notes 

404 ----- 

405 Floating point comparisons are performed by `numpy.allclose`. 

406 """ 

407 d1: ConfigurableActionStruct = getattr(instance1, self.name) 

408 d2: ConfigurableActionStruct = getattr(instance2, self.name) 

409 name = getComparisonName( 

410 _joinNamePath(instance1._name, self.name), 

411 _joinNamePath(instance2._name, self.name) 

412 ) 

413 if not compareScalars(f"keys for {name}", set(d1.fieldNames), set(d2.fieldNames), output=output): 

414 return False 

415 equal = True 

416 for k, v1 in d1.items(): 

417 v2 = getattr(d2, k) 

418 result = compareConfigs(f"{name}.{k}", v1, v2, shortcut=shortcut, 

419 rtol=rtol, atol=atol, output=output) 

420 if not result and shortcut: 

421 return False 

422 equal = equal and result 

423 return equal