Coverage for python/lsst/verify/metricset.py: 17%

160 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-29 02:51 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21__all__ = ['MetricSet'] 

22 

23import os 

24import glob 

25 

26from astropy.table import Table 

27 

28from lsst.utils import getPackageDir 

29from .jsonmixin import JsonSerializationMixin 

30from .metric import Metric 

31from .naming import Name 

32from .yamlutils import load_ordered_yaml 

33 

34 

35class MetricSet(JsonSerializationMixin): 

36 r"""A collection of `Metric`\ s. 

37 

38 Parameters 

39 ---------- 

40 metrics : sequence of `Metric` instances, optional 

41 `Metric`\ s to be contained within the ``MetricSet``. 

42 """ 

43 

44 def __init__(self, metrics=None): 

45 # Internal dict of Metrics. The MetricSet manages access through its 

46 # own mapping API. 

47 self._metrics = {} 

48 

49 if metrics is not None: 

50 for metric in metrics: 

51 if not isinstance(metric, Metric): 

52 message = '{0!r} is not a Metric-type'.format(metric) 

53 raise TypeError(message) 

54 self._metrics[metric.name] = metric 

55 

56 @classmethod 

57 def load_metrics_package(cls, package_name_or_path='verify_metrics', 

58 subset=None): 

59 """Create a MetricSet from a Verification Framework metrics package. 

60 

61 Parameters 

62 ---------- 

63 package_name_or_path : `str`, optional 

64 Name of an EUPS package that hosts metric and specification 

65 definition YAML files **or** the file path to a metrics package. 

66 ``'verify_metrics'`` is the default package, and is where metrics 

67 and specifications are defined for most packages. 

68 subset : `str`, optional 

69 If set, only metrics for this package are loaded. For example, if 

70 ``subset='validate_drp'``, only ``validate_drp`` metrics are 

71 included in the `MetricSet`. This argument is equivalent to the 

72 `MetricSet.subset` method. Default is `None`. 

73 

74 Returns 

75 ------- 

76 metric_set : `MetricSet` 

77 A `MetricSet` containing `Metric` instances. 

78 

79 See also 

80 -------- 

81 lsst.verify.MetricSet.load_single_package 

82 

83 Notes 

84 ----- 

85 EUPS packages that host metrics and specification definitions for the 

86 Verification Framework have top-level directories named ``'metrics'`` 

87 and ``'specs'``. The metrics package chosen with the 

88 ``package_name_or_path`` argument. The default metric package for 

89 LSST Science Pipelines is ``verify_metrics``. 

90 

91 To make a `MetricSet` from a single package's YAML metric definition 

92 file that **is not** contained in a metrics package, 

93 use `load_single_package` instead. 

94 """ 

95 try: 

96 # Try an EUPS package name 

97 package_dir = getPackageDir(package_name_or_path) 

98 except LookupError: 

99 # Try as a filesystem path instead 

100 package_dir = package_name_or_path 

101 finally: 

102 package_dir = os.path.abspath(package_dir) 

103 

104 metrics_dirname = os.path.join(package_dir, 'metrics') 

105 if not os.path.isdir(metrics_dirname): 

106 message = 'Metrics directory {0} not found' 

107 raise OSError(message.format(metrics_dirname)) 

108 

109 metrics = [] 

110 

111 if subset is not None: 

112 # Load only a single package's YAML file 

113 metrics_yaml_paths = [os.path.join(metrics_dirname, 

114 '{0}.yaml'.format(subset))] 

115 else: 

116 # Load all package's YAML files 

117 metrics_yaml_paths = glob.glob(os.path.join(metrics_dirname, 

118 '*.yaml')) 

119 

120 for metrics_yaml_path in metrics_yaml_paths: 

121 new_metrics = MetricSet._load_metrics_yaml(metrics_yaml_path) 

122 metrics.extend(new_metrics) 

123 

124 return cls(metrics) 

125 

126 @classmethod 

127 def load_single_package(cls, metrics_yaml_path): 

128 """Create a MetricSet from a single YAML file containing metric 

129 definitions for a single package. 

130 

131 Returns 

132 ------- 

133 metric_set : `MetricSet` 

134 A `MetricSet` containing `Metric` instances found in the YAML 

135 file. 

136 

137 See also 

138 -------- 

139 lsst.verify.MetricSet.load_metrics_package 

140 

141 Notes 

142 ----- 

143 The YAML file's name, without extension, is taken as the package 

144 name for all metrics. 

145 

146 For example, ``validate_drp.yaml`` contains metrics that are 

147 identified as belonging to the ``validate_drp`` package. 

148 """ 

149 metrics = MetricSet._load_metrics_yaml(metrics_yaml_path) 

150 return cls(metrics) 

151 

152 @staticmethod 

153 def _load_metrics_yaml(metrics_yaml_path): 

154 # package name is inferred from YAML file name (by definition) 

155 metrics_yaml_path = os.path.abspath(metrics_yaml_path) 

156 package_name = os.path.splitext(os.path.basename(metrics_yaml_path))[0] 

157 

158 metrics = [] 

159 with open(metrics_yaml_path) as f: 

160 yaml_doc = load_ordered_yaml(f) 

161 for metric_name, metric_doc in yaml_doc.items(): 

162 name = Name(package=package_name, metric=metric_name) 

163 # throw away a 'name' field if there happens to be one 

164 metric_doc.pop('name', None) 

165 # Create metric instance 

166 metric = Metric.deserialize(name=name, **metric_doc) 

167 metrics.append(metric) 

168 return metrics 

169 

170 @classmethod 

171 def deserialize(cls, metrics=None): 

172 """Deserialize metric JSON objects into a MetricSet instance. 

173 

174 Parameters 

175 ---------- 

176 metrics : `list` 

177 List of metric JSON serializations (typically created by 

178 `MetricSet.json`). 

179 

180 Returns 

181 ------- 

182 metric_set : `MetricSet` 

183 `MetricSet` instance. 

184 """ 

185 instance = cls() 

186 for metric_doc in metrics: 

187 metric = Metric.deserialize(**metric_doc) 

188 instance.insert(metric) 

189 return instance 

190 

191 @property 

192 def json(self): 

193 """A JSON-serializable object (`list`).""" 

194 doc = JsonSerializationMixin._jsonify_list( 

195 [metric for name, metric in self.items()] 

196 ) 

197 return doc 

198 

199 def __getitem__(self, key): 

200 if not isinstance(key, Name): 

201 key = Name(metric=key) 

202 return self._metrics[key] 

203 

204 def __setitem__(self, key, value): 

205 if not isinstance(key, Name): 

206 key = Name(metric=key) 

207 

208 # Key name must be for a metric 

209 if not key.is_metric: 

210 message = 'Key {0!r} is not a metric name'.format(key) 

211 raise KeyError(message) 

212 

213 # value must be a metric type 

214 if not isinstance(value, Metric): 

215 message = 'Expected {0!s}={1!r} to be a Metric-type'.format( 

216 key, value) 

217 raise TypeError(message) 

218 

219 # Metric name and key name must be consistent 

220 if value.name != key: 

221 message = 'Key {0!s} inconsistent with Metric {0!s}' 

222 raise KeyError(message.format(key, value)) 

223 

224 self._metrics[key] = value 

225 

226 def __delitem__(self, key): 

227 if not isinstance(key, Name): 

228 key = Name(metric=key) 

229 del self._metrics[key] 

230 

231 def __len__(self): 

232 return len(self._metrics) 

233 

234 def __contains__(self, key): 

235 if not isinstance(key, Name): 

236 key = Name(metric=key) 

237 return key in self._metrics 

238 

239 def __iter__(self): 

240 for key in self._metrics: 

241 yield key 

242 

243 def __str__(self): 

244 count = len(self) 

245 if count == 0: 

246 count_str = 'empty' 

247 elif count == 1: 

248 count_str = '1 Metric' 

249 else: 

250 count_str = '{count:d} Metrics'.format(count=count) 

251 return '<MetricSet: {0}>'.format(count_str) 

252 

253 def __eq__(self, other): 

254 if len(self) != len(other): 

255 return False 

256 

257 for name, metric in self.items(): 

258 try: 

259 if metric != other[name]: 

260 return False 

261 except KeyError: 

262 return False 

263 

264 return True 

265 

266 def __ne__(self, other): 

267 return not self.__eq__(other) 

268 

269 def __iadd__(self, other): 

270 """Merge another `MetricSet` into this one. 

271 

272 Parameters 

273 --------- 

274 other : `MetricSet` 

275 Another `MetricSet`. Metrics in ``other`` that do exist in this 

276 set are added to this one. Metrics in ``other`` replace metrics of 

277 the same name in this one. 

278 

279 Returns 

280 ------- 

281 self : `MetricSet` 

282 This `MetricSet`. 

283 

284 Notes 

285 ----- 

286 Equivalent to `update`. 

287 """ 

288 self.update(other) 

289 return self 

290 

291 def insert(self, metric): 

292 """Insert a `Metric` into the set. 

293 

294 Any pre-existing metric with the same name is replaced 

295 

296 Parameters 

297 ---------- 

298 metric : `Metric` 

299 A metric. 

300 """ 

301 self[metric.name] = metric 

302 

303 def keys(self): 

304 r"""Get a list of metric names included in the set 

305 

306 Returns 

307 ------- 

308 keys : `list` of `Name` 

309 List of `Name`\ s included in the set. 

310 """ 

311 return self._metrics.keys() 

312 

313 def items(self): 

314 """Iterate over ``(name, metric)`` pairs in the set. 

315 

316 Yields 

317 ------ 

318 item : tuple 

319 Tuple containing: 

320 

321 - `Name` of the `Metric` 

322 - `Metric` instance 

323 """ 

324 for item in self._metrics.items(): 

325 yield item 

326 

327 def subset(self, package=None, tags=None): 

328 """Create a new `MetricSet` with metrics belonging to a single 

329 package and/or tag. 

330 

331 Parameters 

332 ---------- 

333 package : `str` or `lsst.verify.Name`, optional 

334 Name of the package to subset metrics by. If the package name 

335 is ``'pkg_a'``, then metric ``'pkg_a.metric_1'`` would be 

336 **included** in the subset, while ``'pkg_b.metric_2'`` would be 

337 **excluded**. 

338 tags : sequence of `str`, optional 

339 Tags to select metrics by. These tags must be a subset (``<=``) 

340 of the `Metric.tags` for the metric to be selected. 

341 

342 Returns 

343 ------- 

344 metric_subset : `MetricSet` 

345 Subset of this metric set containing only metrics belonging 

346 to the specified package and/or tag. 

347 

348 Notes 

349 ----- 

350 If both ``package`` and ``tag`` are provided then the resulting 

351 `MetricSet` contains the **intersection** of the package-based and 

352 tag-based selections. That is, metrics will belong to ``package`` 

353 and posess the tag ``tag``. 

354 """ 

355 if package is not None and not isinstance(package, Name): 

356 package = Name(package=package) 

357 

358 if tags is not None: 

359 tags = set(tags) 

360 

361 if package is not None and tags is None: 

362 metrics = [metric for metric_name, metric in self._metrics.items() 

363 if metric_name in package] 

364 

365 elif package is not None and tags is not None: 

366 metrics = [metric for metric_name, metric in self._metrics.items() 

367 if metric_name in package 

368 if tags <= metric.tags] 

369 

370 elif package is None and tags is not None: 

371 metrics = [metric for metric_name, metric in self._metrics.items() 

372 if tags <= metric.tags] 

373 

374 else: 

375 metrics = [] 

376 

377 return MetricSet(metrics) 

378 

379 def update(self, other): 

380 """Merge another `MetricSet` into this one. 

381 

382 Parameters 

383 ---------- 

384 other : `MetricSet` 

385 Another `MetricSet`. Metrics in ``other`` that do exist in this 

386 set are added to this one. Metrics in ``other`` replace metrics of 

387 the same name in this one. 

388 """ 

389 for _, metric in other.items(): 

390 self.insert(metric) 

391 

392 def _repr_html_(self): 

393 """Make an HTML representation of metrics for Jupyter notebooks. 

394 """ 

395 name_col = [] 

396 tags_col = [] 

397 units_col = [] 

398 description_col = [] 

399 reference_col = [] 

400 

401 metric_names = list(self.keys()) 

402 metric_names.sort() 

403 

404 for metric_name in metric_names: 

405 metric = self[metric_name] 

406 

407 name_col.append(str(metric_name)) 

408 

409 tags = list(metric.tags) 

410 tags.sort() 

411 tags_col.append(', '.join(tags)) 

412 

413 units_col.append("{0:latex}".format(metric.unit)) 

414 

415 description_col.append(metric.description) 

416 

417 reference_col.append(metric.reference) 

418 

419 table = Table([name_col, description_col, units_col, reference_col, 

420 tags_col], 

421 names=['Name', 'Description', 'Units', 'Reference', 

422 'Tags']) 

423 return table._repr_html_()