Coverage for python/lsst/verify/jobmetadata.py: 23%

88 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-19 12:27 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21__all__ = ['Metadata'] 

22 

23from collections import ChainMap 

24import json 

25import re 

26 

27from .jsonmixin import JsonSerializationMixin 

28 

29 

30class Metadata(JsonSerializationMixin): 

31 """Container for verification framework job metadata. 

32 

33 Metadata are key-value terms. Both keys and values should be 

34 JSON-serializable. 

35 

36 Parameters 

37 ---------- 

38 measurement_set : `lsst.verify.MeasurementSet`, optional 

39 When provided, metadata with keys prefixed by metric names are 

40 deferred to `Metadata` instances attached to measurements 

41 (`lsst.verify.Measurement.notes`). 

42 data : `dict`, optional 

43 Dictionary to seed metadata. 

44 """ 

45 

46 # Pattern for detecting metric name prefixes in names 

47 _prefix_pattern = re.compile(r'^(\S+\.\S+)\.') 

48 

49 def __init__(self, measurement_set, data=None): 

50 

51 # Dict of job metadata not stored with a mesaurement 

52 self._data = {} 

53 

54 # Measurement set to get measurement annotations from 

55 self._meas_set = measurement_set 

56 

57 # Initialize the ChainMap. The first item in the chain map is the 

58 # Metadata object's own _data. This is generic metadata. Additional 

59 # items in the chain are Measurement.notes annotations for all 

60 # measurements in the measurement_set. 

61 self._chain = ChainMap(self._data) 

62 self._cached_prefixes = set() 

63 self._refresh_chainmap() 

64 

65 if data is not None: 

66 self.update(data) 

67 

68 def _refresh_chainmap(self): 

69 prefixes = set([str(name) for name in self._meas_set]) 

70 

71 if self._cached_prefixes != prefixes: 

72 self._cached_prefixes = prefixes 

73 

74 self._chain = ChainMap(self._data) 

75 for _, measurement in self._meas_set.items(): 

76 # Get the dict instance directly so we don't use 

77 # the MeasurementNotes's key auto-prefixing. 

78 self._chain.maps.append(measurement.notes._data) 

79 

80 @staticmethod 

81 def _get_prefix(key): 

82 """Get the prefix of a measurement not, if it exists. 

83 

84 Examples 

85 -------- 

86 >>> Metadata._get_prefix('note') is None 

87 True 

88 >>> Metadata._get_prefix('validate_drp.PA1.note') 

89 'validate_drp.PA1.' 

90 

91 To get the metric name: 

92 

93 >>> prefix = Metadata._get_prefix('validate_drp.PA1.note') 

94 >>> prefix.rstrip('.') 

95 'validate_drp.PA1' 

96 """ 

97 match = Metadata._prefix_pattern.match(key) 

98 if match is not None: 

99 return match.group(0) 

100 else: 

101 return None 

102 

103 def __getitem__(self, key): 

104 self._refresh_chainmap() 

105 return self._chain[key] 

106 

107 def __setitem__(self, key, value): 

108 prefix = Metadata._get_prefix(key) 

109 if prefix is not None: 

110 metric_name = prefix.rstrip('.') 

111 if metric_name in self._meas_set: 

112 # turn prefix into a metric name 

113 self._meas_set[metric_name].notes[key] = value 

114 return 

115 

116 # No matching measurement; insert into general metadata 

117 self._data[key] = value 

118 

119 def __delitem__(self, key): 

120 prefix = Metadata._get_prefix(key) 

121 if prefix is not None: 

122 metric_name = prefix.rstrip('.') 

123 if metric_name in self._meas_set: 

124 del self._meas_set[metric_name].notes[key] 

125 return 

126 

127 # No matching measurement; delete from general metadata 

128 del self._data[key] 

129 

130 def __contains__(self, key): 

131 self._refresh_chainmap() 

132 return key in self._chain 

133 

134 def __len__(self): 

135 self._refresh_chainmap() 

136 return len(self._chain) 

137 

138 def __iter__(self): 

139 self._refresh_chainmap() 

140 for key in self._chain: 

141 yield key 

142 

143 def __eq__(self, other): 

144 # No explicit chain refresh because __len__ already does it 

145 if len(self) != len(other): 

146 return False 

147 

148 for key, value in other.items(): 

149 if key not in self: 

150 return False 

151 if value != self[key]: 

152 return False 

153 

154 return True 

155 

156 def __ne__(self, other): 

157 return not self.__eq__(other) 

158 

159 def __str__(self): 

160 json_data = self.json 

161 return json.dumps(json_data, sort_keys=True, indent=4) 

162 

163 def __repr__(self): 

164 return repr(self._chain) 

165 

166 def _repr_html_(self): 

167 return self.__str__() 

168 

169 def keys(self): 

170 """Get a `list` of metadata keys. 

171 

172 Returns 

173 ------- 

174 keys : `list` of `str` 

175 These keys keys can be used to access metadata values (like a 

176 `dict`). 

177 """ 

178 return [key for key in self] 

179 

180 def items(self): 

181 """Iterate over key-value metadata pairs. 

182 

183 Yields 

184 ------ 

185 item : `tuple` 

186 A metadata item is a tuple of: 

187 

188 - Key (`str`). 

189 - Value (object). 

190 """ 

191 self._refresh_chainmap() 

192 for item in self._chain.items(): 

193 yield item 

194 

195 def update(self, data): 

196 """Update metadata with key-value pairs from a `dict`-like object. 

197 

198 Parameters 

199 ---------- 

200 data : `dict`-like 

201 The ``data`` object needs to provide an ``items`` method to 

202 iterate over its key-value pairs. If this ``Metadata`` instance 

203 already has a key, the value will be overwritten with the value 

204 from ``data``. 

205 """ 

206 for key, value in data.items(): 

207 self[key] = value 

208 

209 @property 

210 def json(self): 

211 """A `dict` that can be serialized as semantic SQUASH JSON. 

212 

213 Keys in the `dict` are metadata keys (see `Metadata.keys`). Values 

214 are the associated metadata values as JSON-serializable objects. 

215 """ 

216 self._refresh_chainmap() 

217 return self.jsonify_dict(self._chain)