Coverage for python/lsst/daf/butler/dimensions/_config.py: 47%

223 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-05 10:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("DimensionConfig",) 

31 

32import textwrap 

33from collections.abc import Mapping, Sequence, Set 

34from typing import Any, ClassVar, Literal, Union, final 

35 

36import pydantic 

37from lsst.resources import ResourcePath, ResourcePathExpression 

38from lsst.sphgeom import PixelizationABC 

39from lsst.utils.doImport import doImportType 

40 

41from .._config import Config, ConfigSubset 

42from .._named import NamedValueSet 

43from .._topology import TopologicalSpace 

44from ._database import DatabaseTopologicalFamilyConstructionVisitor 

45from ._elements import Dimension, KeyColumnSpec, MetadataColumnSpec 

46from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor 

47 

48# The default namespace to use on older dimension config files that only 

49# have a version. 

50_DEFAULT_NAMESPACE = "daf_butler" 

51 

52 

53class DimensionConfig(ConfigSubset): 

54 """Configuration that defines a `DimensionUniverse`. 

55 

56 The configuration tree for dimensions is a (nested) dictionary 

57 with five top-level entries: 

58 

59 - version: an integer version number, used as keys in a singleton registry 

60 of all `DimensionUniverse` instances; 

61 

62 - namespace: a string to be associated with the version in the singleton 

63 registry of all `DimensionUnivers` instances; 

64 

65 - skypix: a dictionary whose entries each define a `SkyPixSystem`, 

66 along with a special "common" key whose value is the name of a skypix 

67 dimension that is used to relate all other spatial dimensions in the 

68 `Registry` database; 

69 

70 - elements: a nested dictionary whose entries each define 

71 `StandardDimension` or `StandardDimensionCombination`. 

72 

73 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys, 

74 with dictionary values that each define a `StandardTopologicalFamily`. 

75 

76 - packers: ignored. 

77 

78 See the documentation for the linked classes above for more information 

79 on the configuration syntax. 

80 

81 Parameters 

82 ---------- 

83 other : `Config` or `str` or `dict`, optional 

84 Argument specifying the configuration information as understood 

85 by `Config`. If `None` is passed then defaults are loaded from 

86 "dimensions.yaml", otherwise defaults are not loaded. 

87 validate : `bool`, optional 

88 If `True` required keys will be checked to ensure configuration 

89 consistency. 

90 searchPaths : `list` or `tuple`, optional 

91 Explicit additional paths to search for defaults. They should 

92 be supplied in priority order. These paths have higher priority 

93 than those read from the environment in 

94 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to 

95 the local file system or URIs, `lsst.resources.ResourcePath`. 

96 """ 

97 

98 requiredKeys = ("version", "elements", "skypix") 

99 defaultConfigFile = "dimensions.yaml" 

100 

101 def __init__( 

102 self, 

103 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None, 

104 validate: bool = True, 

105 searchPaths: Sequence[ResourcePathExpression] | None = None, 

106 ): 

107 # if argument is not None then do not load/merge defaults 

108 mergeDefaults = other is None 

109 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths) 

110 

111 def _updateWithConfigsFromPath( 

112 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str 

113 ) -> None: 

114 """Search the supplied paths reading config from first found. 

115 

116 Raises 

117 ------ 

118 FileNotFoundError 

119 Raised if config file is not found in any of given locations. 

120 

121 Notes 

122 ----- 

123 This method overrides base class method with different behavior. 

124 Instead of merging all found files into a single configuration it 

125 finds first matching file and reads it. 

126 """ 

127 uri = ResourcePath(configFile) 

128 if uri.isabs() and uri.exists(): 

129 # Assume this resource exists 

130 self._updateWithOtherConfigFile(configFile) 

131 self.filesRead.append(configFile) 

132 else: 

133 for pathDir in searchPaths: 

134 if isinstance(pathDir, str | ResourcePath): 

135 pathDir = ResourcePath(pathDir, forceDirectory=True) 

136 file = pathDir.join(configFile) 

137 if file.exists(): 

138 self.filesRead.append(file) 

139 self._updateWithOtherConfigFile(file) 

140 break 

141 else: 

142 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}") 

143 else: 

144 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}") 

145 

146 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None: 

147 """Override for base class method. 

148 

149 Parameters 

150 ---------- 

151 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict` 

152 Entity that can be converted to a `ConfigSubset`. 

153 """ 

154 # Use this class to read the defaults so that subsetting can happen 

155 # correctly. 

156 externalConfig = type(self)(file, validate=False) 

157 self.update(externalConfig) 

158 

159 def makeBuilder(self) -> DimensionConstructionBuilder: 

160 """Construct a `DimensionConstructionBuilder`. 

161 

162 The builder will reflect this configuration. 

163 

164 Returns 

165 ------- 

166 builder : `DimensionConstructionBuilder` 

167 A builder object populated with all visitors from this 

168 configuration. The `~DimensionConstructionBuilder.finish` method 

169 will not have been called. 

170 """ 

171 validated = _UniverseConfig.model_validate(self.toDict()) 

172 builder = DimensionConstructionBuilder( 

173 validated.version, 

174 validated.skypix.common, 

175 self, 

176 namespace=validated.namespace, 

177 ) 

178 for system_name, system_config in sorted(validated.skypix.systems.items()): 

179 builder.add(system_name, system_config) 

180 for element_name, element_config in validated.elements.items(): 

181 builder.add(element_name, element_config) 

182 for family_name, members in validated.topology.spatial.items(): 

183 builder.add( 

184 family_name, 

185 DatabaseTopologicalFamilyConstructionVisitor(space=TopologicalSpace.SPATIAL, members=members), 

186 ) 

187 for family_name, members in validated.topology.temporal.items(): 

188 builder.add( 

189 family_name, 

190 DatabaseTopologicalFamilyConstructionVisitor( 

191 space=TopologicalSpace.TEMPORAL, members=members 

192 ), 

193 ) 

194 return builder 

195 

196 

197@final 

198class _SkyPixSystemConfig(pydantic.BaseModel, DimensionConstructionVisitor): 

199 """Description of a hierarchical sky pixelization system in dimension 

200 universe configuration. 

201 """ 

202 

203 class_: str = pydantic.Field( 

204 alias="class", 

205 description="Fully-qualified name of an `lsst.sphgeom.PixelizationABC implementation.", 

206 ) 

207 

208 min_level: int = 1 

209 """Minimum level for this pixelization.""" 

210 

211 max_level: int | None 

212 """Maximum level for this pixelization.""" 

213 

214 def has_dependencies_in(self, others: Set[str]) -> bool: 

215 # Docstring inherited from DimensionConstructionVisitor. 

216 return False 

217 

218 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None: 

219 # Docstring inherited from DimensionConstructionVisitor. 

220 PixelizationClass = doImportType(self.class_) 

221 assert issubclass(PixelizationClass, PixelizationABC) 

222 if self.max_level is None: 

223 max_level: int | None = getattr(PixelizationClass, "MAX_LEVEL", None) 

224 if max_level is None: 

225 raise TypeError( 

226 f"Skypix pixelization class {self.class_} does" 

227 " not have MAX_LEVEL but no max level has been set explicitly." 

228 ) 

229 self.max_level = max_level 

230 

231 from ._skypix import SkyPixSystem 

232 

233 system = SkyPixSystem( 

234 name, 

235 maxLevel=self.max_level, 

236 PixelizationClass=PixelizationClass, 

237 ) 

238 builder.topology[TopologicalSpace.SPATIAL].add(system) 

239 for level in range(self.min_level, self.max_level + 1): 

240 dimension = system[level] 

241 builder.dimensions.add(dimension) 

242 builder.elements.add(dimension) 

243 

244 

245@final 

246class _SkyPixSectionConfig(pydantic.BaseModel): 

247 """Section of the dimension universe configuration that describes sky 

248 pixelizations. 

249 """ 

250 

251 common: str = pydantic.Field( 

252 description="Name of the dimension used to relate all other spatial dimensions." 

253 ) 

254 

255 systems: dict[str, _SkyPixSystemConfig] = pydantic.Field( 

256 default_factory=dict, description="Descriptions of the supported sky pixelization systems." 

257 ) 

258 

259 model_config = pydantic.ConfigDict(extra="allow") 

260 

261 @pydantic.model_validator(mode="after") 

262 def _move_extra_to_systems(self) -> _SkyPixSectionConfig: 

263 """Reinterpret extra fields in this model as members of the `systems` 

264 dictionary. 

265 """ 

266 if self.__pydantic_extra__ is None: 

267 self.__pydantic_extra__ = {} 

268 for name, data in self.__pydantic_extra__.items(): 

269 self.systems[name] = _SkyPixSystemConfig.model_validate(data) 

270 self.__pydantic_extra__.clear() 

271 return self 

272 

273 

274@final 

275class _TopologySectionConfig(pydantic.BaseModel): 

276 """Section of the dimension universe configuration that describes spatial 

277 and temporal relationships. 

278 """ 

279 

280 spatial: dict[str, list[str]] = pydantic.Field( 

281 default_factory=dict, 

282 description=textwrap.dedent( 

283 """\ 

284 Dictionary of spatial dimension elements, grouped by the "family" 

285 they belong to. 

286 

287 Elements in a family are ordered from fine-grained to coarse-grained. 

288 """ 

289 ), 

290 ) 

291 

292 temporal: dict[str, list[str]] = pydantic.Field( 

293 default_factory=dict, 

294 description=textwrap.dedent( 

295 """\ 

296 Dictionary of temporal dimension elements, grouped by the "family" 

297 they belong to. 

298 

299 Elements in a family are ordered from fine-grained to coarse-grained. 

300 """ 

301 ), 

302 ) 

303 

304 

305@final 

306class _LegacyGovernorDimensionStorage(pydantic.BaseModel): 

307 """Legacy storage configuration for governor dimensions.""" 

308 

309 cls: Literal["lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"] = ( 

310 "lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage" 

311 ) 

312 

313 has_own_table: ClassVar[Literal[True]] = True 

314 """Whether this dimension needs a database table to be defined.""" 

315 

316 is_cached: ClassVar[Literal[True]] = True 

317 """Whether this dimension's records should be cached in clients.""" 

318 

319 implied_union_target: ClassVar[Literal[None]] = None 

320 """Name of another dimension that implies this one, whose values for this 

321 dimension define the set of allowed values for this dimension. 

322 """ 

323 

324 

325@final 

326class _LegacyTableDimensionStorage(pydantic.BaseModel): 

327 """Legacy storage configuration for regular dimension tables stored in the 

328 database. 

329 """ 

330 

331 cls: Literal["lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"] = ( 

332 "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage" 

333 ) 

334 

335 has_own_table: ClassVar[Literal[True]] = True 

336 """Whether this dimension element needs a database table to be defined.""" 

337 

338 is_cached: ClassVar[Literal[False]] = False 

339 """Whether this dimension element's records should be cached in clients.""" 

340 

341 implied_union_target: ClassVar[Literal[None]] = None 

342 """Name of another dimension that implies this one, whose values for this 

343 dimension define the set of allowed values for this dimension. 

344 """ 

345 

346 

347@final 

348class _LegacyImpliedUnionDimensionStorage(pydantic.BaseModel): 

349 """Legacy storage configuration for dimensions whose allowable values are 

350 computed from the union of the values in another dimension that implies 

351 this one. 

352 """ 

353 

354 cls: Literal["lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"] = ( 

355 "lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage" 

356 ) 

357 

358 view_of: str 

359 """The dimension that implies this one and defines its values.""" 

360 

361 has_own_table: ClassVar[Literal[False]] = False 

362 """Whether this dimension needs a database table to be defined.""" 

363 

364 is_cached: ClassVar[Literal[False]] = False 

365 """Whether this dimension element's records should be cached in clients.""" 

366 

367 @property 

368 def implied_union_target(self) -> str: 

369 """Name of another dimension that implies this one, whose values for 

370 this dimension define the set of allowed values for this dimension. 

371 """ 

372 return self.view_of 

373 

374 

375@final 

376class _LegacyCachingDimensionStorage(pydantic.BaseModel): 

377 """Legacy storage configuration that wraps another to indicate that its 

378 records should be cached. 

379 """ 

380 

381 cls: Literal["lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"] = ( 

382 "lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage" 

383 ) 

384 

385 nested: _LegacyTableDimensionStorage | _LegacyImpliedUnionDimensionStorage 

386 """Dimension storage configuration wrapped by this one.""" 

387 

388 @property 

389 def has_own_table(self) -> bool: 

390 """Whether this dimension needs a database table to be defined.""" 

391 return self.nested.has_own_table 

392 

393 is_cached: ClassVar[Literal[True]] = True 

394 """Whether this dimension element's records should be cached in clients.""" 

395 

396 @property 

397 def implied_union_target(self) -> str | None: 

398 """Name of another dimension that implies this one, whose values for 

399 this dimension define the set of allowed values for this dimension. 

400 """ 

401 return self.nested.implied_union_target 

402 

403 

404@final 

405class _ElementConfig(pydantic.BaseModel, DimensionConstructionVisitor): 

406 """Description of a single dimension or dimension join relation in 

407 dimension universe configuration. 

408 """ 

409 

410 doc: str = pydantic.Field(default="", description="Documentation for the dimension or relationship.") 

411 

412 keys: list[KeyColumnSpec] = pydantic.Field( 

413 default_factory=list, 

414 description=textwrap.dedent( 

415 """\ 

416 Key columns that (along with required dependency values) uniquely 

417 identify the dimension's records. 

418 

419 The first columns in this list is the primary key and is used in 

420 data coordinate values. Other columns are alternative keys and 

421 are defined with SQL ``UNIQUE`` constraints. 

422 """ 

423 ), 

424 ) 

425 

426 requires: set[str] = pydantic.Field( 

427 default_factory=set, 

428 description=( 

429 "Other dimensions whose primary keys are part of this dimension element's (compound) primary key." 

430 ), 

431 ) 

432 

433 implies: set[str] = pydantic.Field( 

434 default_factory=set, 

435 description="Other dimensions whose primary keys appear as foreign keys in this dimension element.", 

436 ) 

437 

438 metadata: list[MetadataColumnSpec] = pydantic.Field( 

439 default_factory=list, 

440 description="Non-key columns that provide extra information about a dimension element.", 

441 ) 

442 

443 is_cached: bool = pydantic.Field( 

444 default=False, 

445 description="Whether this element's records should be cached in the client.", 

446 ) 

447 

448 implied_union_target: str | None = pydantic.Field( 

449 default=None, 

450 description=textwrap.dedent( 

451 """\ 

452 Another dimension whose stored values for this dimension form the 

453 set of all allowed values. 

454 

455 The target dimension must have this dimension in its "implies" 

456 list. This means the current dimension will have no table of its 

457 own in the database. 

458 """ 

459 ), 

460 ) 

461 

462 governor: bool = pydantic.Field( 

463 default=False, 

464 description=textwrap.dedent( 

465 """\ 

466 Whether this is a governor dimension. 

467 

468 Governor dimensions are expected to have a tiny number of rows and 

469 must be explicitly provided in any dimension expression in which 

470 dependent dimensions appear. 

471 

472 Implies is_cached=True. 

473 """ 

474 ), 

475 ) 

476 

477 always_join: bool = pydantic.Field( 

478 default=False, 

479 description=textwrap.dedent( 

480 """\ 

481 Whether this dimension join relation should always be included in 

482 any query where its required dependencies appear. 

483 """ 

484 ), 

485 ) 

486 

487 populated_by: str | None = pydantic.Field( 

488 default=None, 

489 description=textwrap.dedent( 

490 """\ 

491 The name of a required dimension that this dimension join 

492 relation's rows should transferred alongside. 

493 """ 

494 ), 

495 ) 

496 

497 storage: Union[ 

498 _LegacyGovernorDimensionStorage, 

499 _LegacyTableDimensionStorage, 

500 _LegacyImpliedUnionDimensionStorage, 

501 _LegacyCachingDimensionStorage, 

502 None, 

503 ] = pydantic.Field( 

504 description="How this dimension element's rows should be stored in the database and client.", 

505 discriminator="cls", 

506 default=None, 

507 ) 

508 

509 def has_dependencies_in(self, others: Set[str]) -> bool: 

510 # Docstring inherited from DimensionConstructionVisitor. 

511 return not (self.requires.isdisjoint(others) and self.implies.isdisjoint(others)) 

512 

513 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None: 

514 # Docstring inherited from DimensionConstructionVisitor. 

515 if self.governor: 

516 from ._governor import GovernorDimension 

517 

518 governor = GovernorDimension( 

519 name, 

520 metadata_columns=NamedValueSet(self.metadata).freeze(), 

521 unique_keys=NamedValueSet(self.keys).freeze(), 

522 doc=self.doc, 

523 ) 

524 builder.dimensions.add(governor) 

525 builder.elements.add(governor) 

526 return 

527 # Expand required dependencies. 

528 for dependency_name in tuple(self.requires): # iterate over copy 

529 self.requires.update(builder.dimensions[dependency_name].required.names) 

530 # Transform required and implied Dimension names into instances, 

531 # and reorder to match builder's order. 

532 required: NamedValueSet[Dimension] = NamedValueSet() 

533 implied: NamedValueSet[Dimension] = NamedValueSet() 

534 for dimension in builder.dimensions: 

535 if dimension.name in self.requires: 

536 required.add(dimension) 

537 if dimension.name in self.implies: 

538 implied.add(dimension) 

539 # Elements with keys are Dimensions; the rest are 

540 # DimensionCombinations. 

541 if self.keys: 

542 from ._database import DatabaseDimension 

543 

544 dimension = DatabaseDimension( 

545 name, 

546 required=required, 

547 implied=implied.freeze(), 

548 metadata_columns=NamedValueSet(self.metadata).freeze(), 

549 unique_keys=NamedValueSet(self.keys).freeze(), 

550 is_cached=self.is_cached, 

551 implied_union_target=self.implied_union_target, 

552 doc=self.doc, 

553 ) 

554 builder.dimensions.add(dimension) 

555 builder.elements.add(dimension) 

556 else: 

557 from ._database import DatabaseDimensionCombination 

558 

559 combination = DatabaseDimensionCombination( 

560 name, 

561 required=required, 

562 implied=implied.freeze(), 

563 doc=self.doc, 

564 metadata_columns=NamedValueSet(self.metadata).freeze(), 

565 is_cached=self.is_cached, 

566 always_join=self.always_join, 

567 populated_by=( 

568 builder.dimensions[self.populated_by] if self.populated_by is not None else None 

569 ), 

570 ) 

571 builder.elements.add(combination) 

572 

573 @pydantic.model_validator(mode="after") 

574 def _primary_key_types(self) -> _ElementConfig: 

575 if self.keys and self.keys[0].type not in ("int", "string"): 

576 raise ValueError( 

577 "The dimension primary key type (the first entry in the keys list) must be 'int' " 

578 f"or 'string'; got '{self.keys[0].type}'." 

579 ) 

580 return self 

581 

582 @pydantic.model_validator(mode="after") 

583 def _not_nullable_keys(self) -> _ElementConfig: 

584 for key in self.keys: 

585 key.nullable = False 

586 return self 

587 

588 @pydantic.model_validator(mode="after") 

589 def _invalid_dimension_fields(self) -> _ElementConfig: 

590 if self.keys: 

591 if self.always_join: 

592 raise ValueError("Dimensions (elements with key columns) may not have always_join=True.") 

593 if self.populated_by: 

594 raise ValueError("Dimensions (elements with key columns) may not have populated_by.") 

595 return self 

596 

597 @pydantic.model_validator(mode="after") 

598 def _storage(self) -> _ElementConfig: 

599 if self.storage is not None: 

600 # 'storage' is legacy; pull its implications into the regular 

601 # attributes and set it to None for consistency. 

602 self.is_cached = self.storage.is_cached 

603 self.implied_union_target = self.storage.implied_union_target 

604 self.storage = None 

605 if self.governor: 

606 self.is_cached = True 

607 if self.implied_union_target is not None: 

608 if self.requires: 

609 raise ValueError("Implied-union dimension may not have required dependencies.") 

610 if self.implies: 

611 raise ValueError("Implied-union dimension may not have implied dependencies.") 

612 if len(self.keys) > 1: 

613 raise ValueError("Implied-union dimension may not have alternate keys.") 

614 if self.metadata: 

615 raise ValueError("Implied-union dimension may not have metadata columns.") 

616 return self 

617 

618 @pydantic.model_validator(mode="after") 

619 def _relationship_dependencies(self) -> _ElementConfig: 

620 if not self.keys and not self.requires: 

621 raise ValueError( 

622 "Dimension relationships (elements with no key columns) must have at least one " 

623 "required dependency." 

624 ) 

625 return self 

626 

627 

628@final 

629class _UniverseConfig(pydantic.BaseModel): 

630 """Configuration that describes a complete dimension data model.""" 

631 

632 version: int = pydantic.Field( 

633 default=0, 

634 description=textwrap.dedent( 

635 """\ 

636 Integer version number for this universe. 

637 

638 This and 'namespace' are expected to uniquely identify a 

639 dimension universe. 

640 """ 

641 ), 

642 ) 

643 

644 namespace: str = pydantic.Field( 

645 default=_DEFAULT_NAMESPACE, 

646 description=textwrap.dedent( 

647 """\ 

648 String namespace for this universe. 

649 

650 This and 'version' are expected to uniquely identify a 

651 dimension universe. 

652 """ 

653 ), 

654 ) 

655 

656 skypix: _SkyPixSectionConfig = pydantic.Field( 

657 description="Hierarchical sky pixelization systems recognized by this dimension universe." 

658 ) 

659 

660 elements: dict[str, _ElementConfig] = pydantic.Field( 

661 default_factory=dict, description="Non-skypix dimensions and dimension join relations." 

662 ) 

663 

664 topology: _TopologySectionConfig = pydantic.Field( 

665 description="Spatial and temporal relationships between dimensions.", 

666 default_factory=_TopologySectionConfig, 

667 )