Coverage for python/lsst/daf/butler/core/composites.py: 25%

54 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:56 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Support for reading and writing composite objects.""" 

23 

24from __future__ import annotations 

25 

26__all__ = ("CompositesConfig", "CompositesMap") 

27 

28import logging 

29from typing import TYPE_CHECKING 

30 

31import yaml 

32 

33from .config import ConfigSubset 

34from .configSupport import processLookupConfigs 

35 

36if TYPE_CHECKING: 

37 from lsst.resources import ResourcePathExpression 

38 

39 from .._butlerConfig import ButlerConfig 

40 from .configSupport import LookupKey 

41 from .datasets import DatasetRef, DatasetType 

42 from .dimensions import DimensionUniverse 

43 from .storageClass import StorageClass 

44 

45log = logging.getLogger(__name__) 

46 

47# Key to access disassembly information 

48DISASSEMBLY_KEY = "disassembled" 

49 

50 

51class CompositesConfig(ConfigSubset): 

52 """Configuration specifics for Composites.""" 

53 

54 component = "composites" 

55 requiredKeys = ("default", DISASSEMBLY_KEY) 

56 defaultConfigFile = "datastores/composites.yaml" 

57 

58 def validate(self) -> None: 

59 """Validate entries have the correct type.""" 

60 super().validate() 

61 # For now assume flat config with keys mapping to booleans 

62 for k, v in self[DISASSEMBLY_KEY].items(): 

63 if not isinstance(v, bool): 

64 raise ValueError(f"CompositesConfig: Key {k} is not a Boolean") 

65 

66 

67class CompositesMap: 

68 """Determine whether something should be disassembled. 

69 

70 Compares a `DatasetType` or `StorageClass` with the map and determines 

71 whether disassembly is requested. 

72 

73 Parameters 

74 ---------- 

75 config : `str`, `ButlerConfig`, or `CompositesConfig` 

76 Configuration to control composites disassembly. 

77 universe : `DimensionUniverse` 

78 Set of all known dimensions, used to expand and validate any used 

79 in lookup keys. 

80 """ 

81 

82 def __init__( 

83 self, config: ResourcePathExpression | ButlerConfig | CompositesConfig, *, universe: DimensionUniverse 

84 ): 

85 if not isinstance(config, CompositesConfig): 

86 config = CompositesConfig(config) 

87 assert isinstance(config, CompositesConfig) 

88 self.config = config 

89 

90 # Pre-filter the disassembly lookup table to remove the 

91 # placeholder __ key we added for documentation. 

92 # It should be harmless but might confuse validation 

93 # Retain the entry as a Config so change in place 

94 disassemblyMap = self.config[DISASSEMBLY_KEY] 

95 for k in set(disassemblyMap): 

96 if k.startswith("__"): 

97 del disassemblyMap[k] 

98 

99 # Calculate the disassembly lookup table -- no need to process 

100 # the values 

101 self._lut = processLookupConfigs(disassemblyMap, universe=universe) 

102 

103 def shouldBeDisassembled(self, entity: DatasetRef | DatasetType | StorageClass) -> bool: 

104 """Indicate whether the entity should be disassembled. 

105 

106 Parameters 

107 ---------- 

108 entity : `StorageClass` or `DatasetType` or `DatasetRef` 

109 Thing to test against the configuration. The ``name`` property 

110 is used to determine a match. A `DatasetType` will first check 

111 its name, before checking its `StorageClass`. If there are no 

112 matches the default will be returned. If the associated 

113 `StorageClass` is not a composite, will always return `False`. 

114 

115 Returns 

116 ------- 

117 disassemble : `bool` 

118 Returns `True` if disassembly should occur; `False` otherwise. 

119 

120 Raises 

121 ------ 

122 ValueError 

123 The supplied argument is not understood. 

124 """ 

125 if not hasattr(entity, "isComposite"): 

126 raise ValueError(f"Supplied entity ({entity}) is not understood.") 

127 

128 # If this is not a composite there is nothing to disassemble. 

129 if not entity.isComposite(): 

130 log.debug("%s will not be disassembled (not a composite)", entity) 

131 return False 

132 

133 matchName: LookupKey | str = f"{entity} (via default)" 

134 disassemble = self.config["default"] 

135 

136 for key in entity._lookupNames(): 

137 if key in self._lut: 

138 disassemble = self._lut[key] 

139 matchName = key 

140 break 

141 

142 if not isinstance(disassemble, bool): 

143 raise TypeError( 

144 f"Got disassemble value {disassemble!r} for config entry {matchName!r}; expected bool." 

145 ) 

146 

147 log.debug("%s will%s be disassembled", matchName, "" if disassemble else " not") 

148 return disassemble 

149 

150 def __str__(self) -> str: 

151 result = {} 

152 result["default"] = self.config["default"] 

153 result["disassembled"] = {} 

154 for key in self._lut: 

155 result["disassembled"][str(key)] = self._lut[key] 

156 return yaml.dump(result)