Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for Storage Classes.""" 

23 

24__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") 

25 

26import builtins 

27import logging 

28 

29from lsst.utils import doImport 

30from .utils import Singleton, getFullTypeName 

31from .assembler import CompositeAssembler 

32from .config import ConfigSubset 

33from .configSupport import LookupKey 

34 

35log = logging.getLogger(__name__) 

36 

37 

38class StorageClassConfig(ConfigSubset): 

39 component = "storageClasses" 

40 defaultConfigFile = "storageClasses.yaml" 

41 

42 

43class StorageClass: 

44 """Class describing how a label maps to a particular Python type. 

45 

46 Parameters 

47 ---------- 

48 name : `str` 

49 Name to use for this class. 

50 pytype : `type` 

51 Python type (or name of type) to associate with the `StorageClass` 

52 components : `dict`, optional 

53 `dict` mapping name of a component to another `StorageClass`. 

54 parameters : `~collections.abc.Sequence` or `~collections.abc.Set` 

55 Parameters understood by this `StorageClass`. 

56 assembler : `str`, optional 

57 Fully qualified name of class supporting assembly and disassembly 

58 of a `pytype` instance. 

59 """ 

60 _cls_name = "BaseStorageClass" 

61 _cls_components = None 

62 _cls_parameters = None 

63 _cls_assembler = None 

64 _cls_pytype = None 

65 defaultAssembler = CompositeAssembler 

66 defaultAssemblerName = getFullTypeName(defaultAssembler) 

67 

68 def __init__(self, name=None, pytype=None, components=None, parameters=None, assembler=None): 

69 if name is None: 

70 name = self._cls_name 

71 if pytype is None: 

72 pytype = self._cls_pytype 

73 if components is None: 

74 components = self._cls_components 

75 if parameters is None: 

76 parameters = self._cls_parameters 

77 if assembler is None: 

78 assembler = self._cls_assembler 

79 self.name = name 

80 

81 if pytype is None: 

82 pytype = object 

83 

84 if not isinstance(pytype, str): 

85 # Already have a type so store it and get the name 

86 self._pytypeName = getFullTypeName(pytype) 

87 self._pytype = pytype 

88 else: 

89 # Store the type name and defer loading of type 

90 self._pytypeName = pytype 

91 

92 self._components = components if components is not None else {} 

93 self._parameters = frozenset(parameters) if parameters is not None else frozenset() 

94 # if the assembler is not None also set it and clear the default 

95 # assembler 

96 if assembler is not None: 

97 self._assemblerClassName = assembler 

98 self._assembler = None 

99 elif components is not None: 

100 # We set a default assembler for composites so that a class is 

101 # guaranteed to support something if it is a composite. 

102 log.debug("Setting default assembler for %s", self.name) 

103 self._assembler = self.defaultAssembler 

104 self._assemblerClassName = self.defaultAssemblerName 

105 else: 

106 self._assembler = None 

107 self._assemblerClassName = None 

108 # The types are created on demand and cached 

109 self._pytype = None 

110 

111 @property 

112 def components(self): 

113 """Component names mapped to associated `StorageClass` 

114 """ 

115 return self._components 

116 

117 @property 

118 def parameters(self): 

119 """`set` of names of parameters supported by this `StorageClass` 

120 """ 

121 return set(self._parameters) 

122 

123 @property 

124 def pytype(self): 

125 """Python type associated with this `StorageClass`.""" 

126 if self._pytype is not None: 

127 return self._pytype 

128 

129 if hasattr(builtins, self._pytypeName): 

130 pytype = getattr(builtins, self._pytypeName) 

131 else: 

132 pytype = doImport(self._pytypeName) 

133 self._pytype = pytype 

134 return self._pytype 

135 

136 @property 

137 def assemblerClass(self): 

138 """Class to use to (dis)assemble an object from components.""" 

139 if self._assembler is not None: 

140 return self._assembler 

141 if self._assemblerClassName is None: 

142 return None 

143 self._assembler = doImport(self._assemblerClassName) 

144 return self._assembler 

145 

146 def assembler(self): 

147 """Return an instance of an assembler. 

148 

149 Returns 

150 ------- 

151 assembler : `CompositeAssembler` 

152 Instance of the assembler associated with this `StorageClass`. 

153 Assembler is constructed with this `StorageClass`. 

154 

155 Raises 

156 ------ 

157 TypeError 

158 This StorageClass has no associated assembler. 

159 """ 

160 cls = self.assemblerClass 

161 if cls is None: 

162 raise TypeError(f"No assembler class is associated with StorageClass {self.name}") 

163 return cls(storageClass=self) 

164 

165 def isComposite(self): 

166 """Boolean indicating whether this `StorageClass` is a composite 

167 or not. 

168 

169 Returns 

170 ------- 

171 isComposite : `bool` 

172 `True` if this `StorageClass` is a composite, `False` 

173 otherwise. 

174 """ 

175 if self.components: 

176 return True 

177 return False 

178 

179 def _lookupNames(self): 

180 """Keys to use when looking up this DatasetRef in a configuration. 

181 

182 The names are returned in order of priority. 

183 

184 Returns 

185 ------- 

186 names : `tuple` of `LookupKey` 

187 Tuple of a `LookupKey` using the `StorageClass` name. 

188 """ 

189 return (LookupKey(name=self.name), ) 

190 

191 def knownParameters(self): 

192 """Return set of all parameters known to this `StorageClass` 

193 

194 The set includes parameters understood by components of a composite. 

195 

196 Returns 

197 ------- 

198 known : `set` 

199 All parameter keys of this `StorageClass` and the component 

200 storage classes. 

201 """ 

202 known = set(self._parameters) 

203 for sc in self.components.values(): 

204 known.update(sc.knownParameters()) 

205 return known 

206 

207 def validateParameters(self, parameters=None): 

208 """Check that the parameters are known to this `StorageClass` 

209 

210 Does not check the values. 

211 

212 Parameters 

213 ---------- 

214 parameters : `~collections.abc.Collection`, optional 

215 Collection containing the parameters. Can be `dict`-like or 

216 `set`-like. The parameter values are not checked. 

217 If no parameters are supplied, always returns without error. 

218 

219 Raises 

220 ------ 

221 KeyError 

222 Some parameters are not understood by this `StorageClass`. 

223 """ 

224 # No parameters is always okay 

225 if not parameters: 

226 return 

227 

228 # Extract the important information into a set. Works for dict and 

229 # list. 

230 external = set(parameters) 

231 

232 diff = external - self.knownParameters() 

233 if diff: 

234 s = "s" if len(diff) > 1 else "" 

235 unknown = '\', \''.join(diff) 

236 raise KeyError(f"Parameter{s} '{unknown}' not understood by StorageClass {self.name}") 

237 

238 def filterParameters(self, parameters, subset=None): 

239 """Filter out parameters that are not known to this StorageClass 

240 

241 Parameters 

242 ---------- 

243 parameters : `dict`, optional 

244 Candidate parameters. Can be `None` if no parameters have 

245 been provided. 

246 subset : `~collections.abc.Collection`, optional 

247 Subset of supported parameters that the caller is interested 

248 in using. The subset must be known to the `StorageClass` 

249 if specified. 

250 

251 Returns 

252 ------- 

253 filtered : `dict` 

254 Valid parameters. Empty `dict` if none are suitable. 

255 """ 

256 if not parameters: 

257 return {} 

258 subset = set(subset) if subset is not None else set() 

259 known = self.knownParameters() 

260 if not subset.issubset(known): 

261 raise ValueError(f"Requested subset ({subset}) is not a subset of" 

262 f" known parameters ({known})") 

263 return {k: parameters[k] for k in known if k in parameters} 

264 

265 def validateInstance(self, instance): 

266 """Check that the supplied Python object has the expected Python type 

267 

268 Parameters 

269 ---------- 

270 instance : `object` 

271 Object to check. 

272 

273 Returns 

274 ------- 

275 isOk : `bool` 

276 True if the supplied instance object can be handled by this 

277 `StorageClass`, False otherwise. 

278 """ 

279 return isinstance(instance, self.pytype) 

280 

281 def __eq__(self, other): 

282 """Equality checks name, pytype name, assembler name, and components""" 

283 if self.name != other.name: 

284 return False 

285 

286 if not isinstance(other, StorageClass): 

287 return False 

288 

289 # We must compare pytype and assembler by name since we do not want 

290 # to trigger an import of external module code here 

291 if self._assemblerClassName != other._assemblerClassName: 

292 return False 

293 if self._pytypeName != other._pytypeName: 

294 return False 

295 

296 # Ensure we have the same component keys in each 

297 if set(self.components.keys()) != set(other.components.keys()): 

298 return False 

299 

300 # Same parameters 

301 if self.parameters != other.parameters: 

302 return False 

303 

304 # Ensure that all the components have the same type 

305 for k in self.components: 

306 if self.components[k] != other.components[k]: 

307 return False 

308 

309 # If we got to this point everything checks out 

310 return True 

311 

312 def __hash__(self): 

313 return hash(self.name) 

314 

315 def __repr__(self): 

316 optionals = {} 

317 if self._pytypeName != "object": 

318 optionals["pytype"] = self._pytypeName 

319 if self._assemblerClassName is not None: 

320 optionals["assembler"] = self._assemblerClassName 

321 if self._parameters: 

322 optionals["parameters"] = self._parameters 

323 if self.components: 

324 optionals["components"] = self.components 

325 

326 # order is preserved in the dict 

327 options = ", ".join(f"{k}={v!r}" for k, v in optionals.items()) 

328 

329 # Start with mandatory fields 

330 r = f"{self.__class__.__name__}({self.name!r}" 

331 if options: 

332 r = r + ", " + options 

333 r = r + ")" 

334 return r 

335 

336 def __str__(self): 

337 return self.name 

338 

339 

340class StorageClassFactory(metaclass=Singleton): 

341 """Factory for `StorageClass` instances. 

342 

343 This class is a singleton, with each instance sharing the pool of 

344 StorageClasses. Since code can not know whether it is the first 

345 time the instance has been created, the constructor takes no arguments. 

346 To populate the factory with storage classes, a call to 

347 `~StorageClassFactory.addFromConfig()` should be made. 

348 

349 Parameters 

350 ---------- 

351 config : `StorageClassConfig` or `str`, optional 

352 Load configuration. In a ButlerConfig` the relevant configuration 

353 is located in the ``storageClasses`` section. 

354 """ 

355 

356 def __init__(self, config=None): 

357 self._storageClasses = {} 

358 self._configs = [] 

359 

360 # Always seed with the default config 

361 self.addFromConfig(StorageClassConfig()) 

362 

363 if config is not None: 

364 self.addFromConfig(config) 

365 

366 def __str__(self): 

367 """Return summary of factory. 

368 

369 Returns 

370 ------- 

371 summary : `str` 

372 Summary of the factory status. 

373 """ 

374 sep = "\n" 

375 return f"""Number of registered StorageClasses: {len(self._storageClasses)} 

376 

377StorageClasses 

378-------------- 

379{sep.join(f"{s}: {self._storageClasses[s]}" for s in self._storageClasses)} 

380""" 

381 

382 def __contains__(self, storageClassOrName): 

383 """Indicates whether the storage class exists in the factory. 

384 

385 Parameters 

386 ---------- 

387 storageClassOrName : `str` or `StorageClass` 

388 If `str` is given existence of the named StorageClass 

389 in the factory is checked. If `StorageClass` is given 

390 existence and equality are checked. 

391 

392 Returns 

393 ------- 

394 in : `bool` 

395 True if the supplied string is present, or if the supplied 

396 `StorageClass` is present and identical. 

397 

398 Notes 

399 ----- 

400 The two different checks (one for "key" and one for "value") based on 

401 the type of the given argument mean that it is possible for 

402 StorageClass.name to be in the factory but StorageClass to not be 

403 in the factory. 

404 """ 

405 if isinstance(storageClassOrName, str): 

406 return storageClassOrName in self._storageClasses 

407 elif isinstance(storageClassOrName, StorageClass): 

408 if storageClassOrName.name in self._storageClasses: 

409 return storageClassOrName == self._storageClasses[storageClassOrName.name] 

410 return False 

411 

412 def addFromConfig(self, config): 

413 """Add more `StorageClass` definitions from a config file. 

414 

415 Parameters 

416 ---------- 

417 config : `StorageClassConfig`, `Config` or `str` 

418 Storage class configuration. Can contain a ``storageClasses`` 

419 key if part of a global configuration. 

420 """ 

421 sconfig = StorageClassConfig(config) 

422 self._configs.append(sconfig) 

423 

424 # Since we can not assume that we will get definitions of 

425 # components or parents before their classes are defined 

426 # we have a helper function that we can call recursively 

427 # to extract definitions from the configuration. 

428 def processStorageClass(name, sconfig): 

429 # Maybe we've already processed this through recursion 

430 if name not in sconfig: 

431 return 

432 info = sconfig.pop(name) 

433 

434 # Always create the storage class so we can ensure that 

435 # we are not trying to overwrite with a different definition 

436 components = None 

437 if "components" in info: 

438 components = {} 

439 for cname, ctype in info["components"].items(): 

440 if ctype not in self: 

441 processStorageClass(ctype, sconfig) 

442 components[cname] = self.getStorageClass(ctype) 

443 

444 # Extract scalar items from dict that are needed for 

445 # StorageClass Constructor 

446 storageClassKwargs = {k: info[k] for k in ("pytype", "assembler", "parameters") if k in info} 

447 

448 # Fill in other items 

449 storageClassKwargs["components"] = components 

450 

451 # Create the new storage class and register it 

452 baseClass = None 

453 if "inheritsFrom" in info: 

454 baseName = info["inheritsFrom"] 

455 if baseName not in self: 

456 processStorageClass(baseName, sconfig) 

457 baseClass = type(self.getStorageClass(baseName)) 

458 

459 newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs) 

460 newStorageClass = newStorageClassType() 

461 self.registerStorageClass(newStorageClass) 

462 

463 for name in list(sconfig.keys()): 

464 processStorageClass(name, sconfig) 

465 

466 @staticmethod 

467 def makeNewStorageClass(name, baseClass=StorageClass, **kwargs): 

468 """Create a new Python class as a subclass of `StorageClass`. 

469 

470 Parameters 

471 ---------- 

472 name : `str` 

473 Name to use for this class. 

474 baseClass : `type`, optional 

475 Base class for this `StorageClass`. Must be either `StorageClass` 

476 or a subclass of `StorageClass`. If `None`, `StorageClass` will 

477 be used. 

478 

479 Returns 

480 ------- 

481 newtype : `type` subclass of `StorageClass` 

482 Newly created Python type. 

483 """ 

484 

485 if baseClass is None: 

486 baseClass = StorageClass 

487 if not issubclass(baseClass, StorageClass): 

488 raise ValueError(f"Base class must be a StorageClass not {baseClass}") 

489 

490 # convert the arguments to use different internal names 

491 clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None} 

492 clsargs["_cls_name"] = name 

493 

494 # Some container items need to merge with the base class values 

495 # so that a child can inherit but override one bit. 

496 # lists (which you get from configs) are treated as sets for this to 

497 # work consistently. 

498 for k in ("components", "parameters"): 

499 classKey = f"_cls_{k}" 

500 if classKey in clsargs: 

501 baseValue = getattr(baseClass, classKey, None) 

502 if baseValue is not None: 

503 currentValue = clsargs[classKey] 

504 if isinstance(currentValue, dict): 

505 newValue = baseValue.copy() 

506 else: 

507 newValue = set(baseValue) 

508 newValue.update(currentValue) 

509 clsargs[classKey] = newValue 

510 

511 # If we have parameters they should be a frozen set so that the 

512 # parameters in the class can not be modified. 

513 pk = f"_cls_parameters" 

514 if pk in clsargs: 

515 clsargs[pk] = frozenset(clsargs[pk]) 

516 

517 return type(f"StorageClass{name}", (baseClass,), clsargs) 

518 

519 def getStorageClass(self, storageClassName): 

520 """Get a StorageClass instance associated with the supplied name. 

521 

522 Parameters 

523 ---------- 

524 storageClassName : `str` 

525 Name of the storage class to retrieve. 

526 

527 Returns 

528 ------- 

529 instance : `StorageClass` 

530 Instance of the correct `StorageClass`. 

531 

532 Raises 

533 ------ 

534 KeyError 

535 The requested storage class name is not registered. 

536 """ 

537 return self._storageClasses[storageClassName] 

538 

539 def registerStorageClass(self, storageClass): 

540 """Store the `StorageClass` in the factory. 

541 

542 Will be indexed by `StorageClass.name` and will return instances 

543 of the supplied `StorageClass`. 

544 

545 Parameters 

546 ---------- 

547 storageClass : `StorageClass` 

548 Type of the Python `StorageClass` to register. 

549 

550 Raises 

551 ------ 

552 ValueError 

553 If a storage class has already been registered with 

554 storageClassName and the previous definition differs. 

555 """ 

556 if storageClass.name in self._storageClasses: 

557 existing = self.getStorageClass(storageClass.name) 

558 if existing != storageClass: 

559 raise ValueError(f"New definition for StorageClass {storageClass.name} ({storageClass}) " 

560 f"differs from current definition ({existing})") 

561 else: 

562 self._storageClasses[storageClass.name] = storageClass 

563 

564 def _unregisterStorageClass(self, storageClassName): 

565 """Remove the named StorageClass from the factory. 

566 

567 Parameters 

568 ---------- 

569 storageClassName : `str` 

570 Name of storage class to remove. 

571 

572 Raises 

573 ------ 

574 KeyError 

575 The named storage class is not registered. 

576 

577 Notes 

578 ----- 

579 This method is intended to simplify testing of StorageClassFactory 

580 functionality and it is not expected to be required for normal usage. 

581 """ 

582 del self._storageClasses[storageClassName]