Coverage for python/lsst/daf/butler/datastore/composites.py: 25%

54 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 10:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for reading and writing composite objects.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ("CompositesConfig", "CompositesMap") 

33 

34import logging 

35from typing import TYPE_CHECKING 

36 

37import yaml 

38 

39from .._config import ConfigSubset 

40from .._config_support import processLookupConfigs 

41 

42if TYPE_CHECKING: 

43 from lsst.resources import ResourcePathExpression 

44 

45 from .._butler_config import ButlerConfig 

46 from .._config_support import LookupKey 

47 from .._dataset_ref import DatasetRef 

48 from .._dataset_type import DatasetType 

49 from .._storage_class import StorageClass 

50 from ..dimensions import DimensionUniverse 

51 

52log = logging.getLogger(__name__) 

53 

54# Key to access disassembly information 

55DISASSEMBLY_KEY = "disassembled" 

56 

57 

58class CompositesConfig(ConfigSubset): 

59 """Configuration specifics for Composites.""" 

60 

61 component = "composites" 

62 requiredKeys = ("default", DISASSEMBLY_KEY) 

63 defaultConfigFile = "datastores/composites.yaml" 

64 

65 def validate(self) -> None: 

66 """Validate entries have the correct type.""" 

67 super().validate() 

68 # For now assume flat config with keys mapping to booleans 

69 for k, v in self[DISASSEMBLY_KEY].items(): 

70 if not isinstance(v, bool): 

71 raise ValueError(f"CompositesConfig: Key {k} is not a Boolean") 

72 

73 

74class CompositesMap: 

75 """Determine whether something should be disassembled. 

76 

77 Compares a `DatasetType` or `StorageClass` with the map and determines 

78 whether disassembly is requested. 

79 

80 Parameters 

81 ---------- 

82 config : `str`, `ButlerConfig`, or `CompositesConfig` 

83 Configuration to control composites disassembly. 

84 universe : `DimensionUniverse` 

85 Set of all known dimensions, used to expand and validate any used 

86 in lookup keys. 

87 """ 

88 

89 def __init__( 

90 self, config: ResourcePathExpression | ButlerConfig | CompositesConfig, *, universe: DimensionUniverse 

91 ): 

92 if not isinstance(config, CompositesConfig): 

93 config = CompositesConfig(config) 

94 assert isinstance(config, CompositesConfig) 

95 self.config = config 

96 

97 # Pre-filter the disassembly lookup table to remove the 

98 # placeholder __ key we added for documentation. 

99 # It should be harmless but might confuse validation 

100 # Retain the entry as a Config so change in place 

101 disassemblyMap = self.config[DISASSEMBLY_KEY] 

102 for k in set(disassemblyMap): 

103 if k.startswith("__"): 

104 del disassemblyMap[k] 

105 

106 # Calculate the disassembly lookup table -- no need to process 

107 # the values 

108 self._lut = processLookupConfigs(disassemblyMap, universe=universe) 

109 

110 def shouldBeDisassembled(self, entity: DatasetRef | DatasetType | StorageClass) -> bool: 

111 """Indicate whether the entity should be disassembled. 

112 

113 Parameters 

114 ---------- 

115 entity : `StorageClass` or `DatasetType` or `DatasetRef` 

116 Thing to test against the configuration. The ``name`` property 

117 is used to determine a match. A `DatasetType` will first check 

118 its name, before checking its `StorageClass`. If there are no 

119 matches the default will be returned. If the associated 

120 `StorageClass` is not a composite, will always return `False`. 

121 

122 Returns 

123 ------- 

124 disassemble : `bool` 

125 Returns `True` if disassembly should occur; `False` otherwise. 

126 

127 Raises 

128 ------ 

129 ValueError 

130 The supplied argument is not understood. 

131 """ 

132 if not hasattr(entity, "isComposite"): 

133 raise ValueError(f"Supplied entity ({entity}) is not understood.") 

134 

135 # If this is not a composite there is nothing to disassemble. 

136 if not entity.isComposite(): 

137 log.debug("%s will not be disassembled (not a composite)", entity) 

138 return False 

139 

140 matchName: LookupKey | str = f"{entity} (via default)" 

141 disassemble = self.config["default"] 

142 

143 for key in entity._lookupNames(): 

144 if key in self._lut: 

145 disassemble = self._lut[key] 

146 matchName = key 

147 break 

148 

149 if not isinstance(disassemble, bool): 

150 raise TypeError( 

151 f"Got disassemble value {disassemble!r} for config entry {matchName!r}; expected bool." 

152 ) 

153 

154 log.debug("%s will%s be disassembled", matchName, "" if disassemble else " not") 

155 return disassemble 

156 

157 def __str__(self) -> str: 

158 result = {} 

159 result["default"] = self.config["default"] 

160 result["disassembled"] = {} 

161 for key in self._lut: 

162 result["disassembled"][str(key)] = self._lut[key] 

163 return yaml.dump(result)