Coverage for python/lsst/verify/metricset.py: 16%
160 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:15 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:15 -0700
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ['MetricSet']
23import os
24import glob
26from astropy.table import Table
28from lsst.utils import getPackageDir
29from .jsonmixin import JsonSerializationMixin
30from .metric import Metric
31from .naming import Name
32from .yamlutils import load_ordered_yaml
35class MetricSet(JsonSerializationMixin):
36 r"""A collection of `Metric`\ s.
38 Parameters
39 ----------
40 metrics : sequence of `Metric` instances, optional
41 `Metric`\ s to be contained within the ``MetricSet``.
42 """
44 def __init__(self, metrics=None):
45 # Internal dict of Metrics. The MetricSet manages access through its
46 # own mapping API.
47 self._metrics = {}
49 if metrics is not None:
50 for metric in metrics:
51 if not isinstance(metric, Metric):
52 message = '{0!r} is not a Metric-type'.format(metric)
53 raise TypeError(message)
54 self._metrics[metric.name] = metric
56 @classmethod
57 def load_metrics_package(cls, package_name_or_path='verify_metrics',
58 subset=None):
59 """Create a MetricSet from a Verification Framework metrics package.
61 Parameters
62 ----------
63 package_name_or_path : `str`, optional
64 Name of an EUPS package that hosts metric and specification
65 definition YAML files **or** the file path to a metrics package.
66 ``'verify_metrics'`` is the default package, and is where metrics
67 and specifications are defined for most packages.
68 subset : `str`, optional
69 If set, only metrics for this package are loaded. For example, if
70 ``subset='validate_drp'``, only ``validate_drp`` metrics are
71 included in the `MetricSet`. This argument is equivalent to the
72 `MetricSet.subset` method. Default is `None`.
74 Returns
75 -------
76 metric_set : `MetricSet`
77 A `MetricSet` containing `Metric` instances.
79 See also
80 --------
81 lsst.verify.MetricSet.load_single_package
83 Notes
84 -----
85 EUPS packages that host metrics and specification definitions for the
86 Verification Framework have top-level directories named ``'metrics'``
87 and ``'specs'``. The metrics package chosen with the
88 ``package_name_or_path`` argument. The default metric package for
89 LSST Science Pipelines is ``verify_metrics``.
91 To make a `MetricSet` from a single package's YAML metric definition
92 file that **is not** contained in a metrics package,
93 use `load_single_package` instead.
94 """
95 try:
96 # Try an EUPS package name
97 package_dir = getPackageDir(package_name_or_path)
98 except LookupError:
99 # Try as a filesystem path instead
100 package_dir = package_name_or_path
101 finally:
102 package_dir = os.path.abspath(package_dir)
104 metrics_dirname = os.path.join(package_dir, 'metrics')
105 if not os.path.isdir(metrics_dirname):
106 message = 'Metrics directory {0} not found'
107 raise OSError(message.format(metrics_dirname))
109 metrics = []
111 if subset is not None:
112 # Load only a single package's YAML file
113 metrics_yaml_paths = [os.path.join(metrics_dirname,
114 '{0}.yaml'.format(subset))]
115 else:
116 # Load all package's YAML files
117 metrics_yaml_paths = glob.glob(os.path.join(metrics_dirname,
118 '*.yaml'))
120 for metrics_yaml_path in metrics_yaml_paths:
121 new_metrics = MetricSet._load_metrics_yaml(metrics_yaml_path)
122 metrics.extend(new_metrics)
124 return cls(metrics)
126 @classmethod
127 def load_single_package(cls, metrics_yaml_path):
128 """Create a MetricSet from a single YAML file containing metric
129 definitions for a single package.
131 Returns
132 -------
133 metric_set : `MetricSet`
134 A `MetricSet` containing `Metric` instances found in the YAML
135 file.
137 See also
138 --------
139 lsst.verify.MetricSet.load_metrics_package
141 Notes
142 -----
143 The YAML file's name, without extension, is taken as the package
144 name for all metrics.
146 For example, ``validate_drp.yaml`` contains metrics that are
147 identified as belonging to the ``validate_drp`` package.
148 """
149 metrics = MetricSet._load_metrics_yaml(metrics_yaml_path)
150 return cls(metrics)
152 @staticmethod
153 def _load_metrics_yaml(metrics_yaml_path):
154 # package name is inferred from YAML file name (by definition)
155 metrics_yaml_path = os.path.abspath(metrics_yaml_path)
156 package_name = os.path.splitext(os.path.basename(metrics_yaml_path))[0]
158 metrics = []
159 with open(metrics_yaml_path) as f:
160 yaml_doc = load_ordered_yaml(f)
161 for metric_name, metric_doc in yaml_doc.items():
162 name = Name(package=package_name, metric=metric_name)
163 # throw away a 'name' field if there happens to be one
164 metric_doc.pop('name', None)
165 # Create metric instance
166 metric = Metric.deserialize(name=name, **metric_doc)
167 metrics.append(metric)
168 return metrics
170 @classmethod
171 def deserialize(cls, metrics=None):
172 """Deserialize metric JSON objects into a MetricSet instance.
174 Parameters
175 ----------
176 metrics : `list`
177 List of metric JSON serializations (typically created by
178 `MetricSet.json`).
180 Returns
181 -------
182 metric_set : `MetricSet`
183 `MetricSet` instance.
184 """
185 instance = cls()
186 for metric_doc in metrics:
187 metric = Metric.deserialize(**metric_doc)
188 instance.insert(metric)
189 return instance
191 @property
192 def json(self):
193 """A JSON-serializable object (`list`)."""
194 doc = JsonSerializationMixin._jsonify_list(
195 [metric for name, metric in self.items()]
196 )
197 return doc
199 def __getitem__(self, key):
200 if not isinstance(key, Name):
201 key = Name(metric=key)
202 return self._metrics[key]
204 def __setitem__(self, key, value):
205 if not isinstance(key, Name):
206 key = Name(metric=key)
208 # Key name must be for a metric
209 if not key.is_metric:
210 message = 'Key {0!r} is not a metric name'.format(key)
211 raise KeyError(message)
213 # value must be a metric type
214 if not isinstance(value, Metric):
215 message = 'Expected {0!s}={1!r} to be a Metric-type'.format(
216 key, value)
217 raise TypeError(message)
219 # Metric name and key name must be consistent
220 if value.name != key:
221 message = 'Key {0!s} inconsistent with Metric {0!s}'
222 raise KeyError(message.format(key, value))
224 self._metrics[key] = value
226 def __delitem__(self, key):
227 if not isinstance(key, Name):
228 key = Name(metric=key)
229 del self._metrics[key]
231 def __len__(self):
232 return len(self._metrics)
234 def __contains__(self, key):
235 if not isinstance(key, Name):
236 key = Name(metric=key)
237 return key in self._metrics
239 def __iter__(self):
240 for key in self._metrics:
241 yield key
243 def __str__(self):
244 count = len(self)
245 if count == 0:
246 count_str = 'empty'
247 elif count == 1:
248 count_str = '1 Metric'
249 else:
250 count_str = '{count:d} Metrics'.format(count=count)
251 return '<MetricSet: {0}>'.format(count_str)
253 def __eq__(self, other):
254 if len(self) != len(other):
255 return False
257 for name, metric in self.items():
258 try:
259 if metric != other[name]:
260 return False
261 except KeyError:
262 return False
264 return True
266 def __ne__(self, other):
267 return not self.__eq__(other)
269 def __iadd__(self, other):
270 """Merge another `MetricSet` into this one.
272 Parameters
273 ---------
274 other : `MetricSet`
275 Another `MetricSet`. Metrics in ``other`` that do exist in this
276 set are added to this one. Metrics in ``other`` replace metrics of
277 the same name in this one.
279 Returns
280 -------
281 self : `MetricSet`
282 This `MetricSet`.
284 Notes
285 -----
286 Equivalent to `update`.
287 """
288 self.update(other)
289 return self
291 def insert(self, metric):
292 """Insert a `Metric` into the set.
294 Any pre-existing metric with the same name is replaced
296 Parameters
297 ----------
298 metric : `Metric`
299 A metric.
300 """
301 self[metric.name] = metric
303 def keys(self):
304 r"""Get a list of metric names included in the set
306 Returns
307 -------
308 keys : `list` of `Name`
309 List of `Name`\ s included in the set.
310 """
311 return self._metrics.keys()
313 def items(self):
314 """Iterate over ``(name, metric)`` pairs in the set.
316 Yields
317 ------
318 item : tuple
319 Tuple containing:
321 - `Name` of the `Metric`
322 - `Metric` instance
323 """
324 for item in self._metrics.items():
325 yield item
327 def subset(self, package=None, tags=None):
328 """Create a new `MetricSet` with metrics belonging to a single
329 package and/or tag.
331 Parameters
332 ----------
333 package : `str` or `lsst.verify.Name`, optional
334 Name of the package to subset metrics by. If the package name
335 is ``'pkg_a'``, then metric ``'pkg_a.metric_1'`` would be
336 **included** in the subset, while ``'pkg_b.metric_2'`` would be
337 **excluded**.
338 tags : sequence of `str`, optional
339 Tags to select metrics by. These tags must be a subset (``<=``)
340 of the `Metric.tags` for the metric to be selected.
342 Returns
343 -------
344 metric_subset : `MetricSet`
345 Subset of this metric set containing only metrics belonging
346 to the specified package and/or tag.
348 Notes
349 -----
350 If both ``package`` and ``tag`` are provided then the resulting
351 `MetricSet` contains the **intersection** of the package-based and
352 tag-based selections. That is, metrics will belong to ``package``
353 and posess the tag ``tag``.
354 """
355 if package is not None and not isinstance(package, Name):
356 package = Name(package=package)
358 if tags is not None:
359 tags = set(tags)
361 if package is not None and tags is None:
362 metrics = [metric for metric_name, metric in self._metrics.items()
363 if metric_name in package]
365 elif package is not None and tags is not None:
366 metrics = [metric for metric_name, metric in self._metrics.items()
367 if metric_name in package
368 if tags <= metric.tags]
370 elif package is None and tags is not None:
371 metrics = [metric for metric_name, metric in self._metrics.items()
372 if tags <= metric.tags]
374 else:
375 metrics = []
377 return MetricSet(metrics)
379 def update(self, other):
380 """Merge another `MetricSet` into this one.
382 Parameters
383 ----------
384 other : `MetricSet`
385 Another `MetricSet`. Metrics in ``other`` that do exist in this
386 set are added to this one. Metrics in ``other`` replace metrics of
387 the same name in this one.
388 """
389 for _, metric in other.items():
390 self.insert(metric)
392 def _repr_html_(self):
393 """Make an HTML representation of metrics for Jupyter notebooks.
394 """
395 name_col = []
396 tags_col = []
397 units_col = []
398 description_col = []
399 reference_col = []
401 metric_names = list(self.keys())
402 metric_names.sort()
404 for metric_name in metric_names:
405 metric = self[metric_name]
407 name_col.append(str(metric_name))
409 tags = list(metric.tags)
410 tags.sort()
411 tags_col.append(', '.join(tags))
413 units_col.append("{0:latex}".format(metric.unit))
415 description_col.append(metric.description)
417 reference_col.append(metric.reference)
419 table = Table([name_col, description_col, units_col, reference_col,
420 tags_col],
421 names=['Name', 'Description', 'Units', 'Reference',
422 'Tags'])
423 return table._repr_html_()