Coverage for python/lsst/verify/metricset.py: 77%
161 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-14 18:37 +0000
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ['MetricSet']
23from astropy.table import Table
25from lsst.resources import ResourcePath
26from .jsonmixin import JsonSerializationMixin
27from .metric import Metric
28from .naming import Name
29from .yamlutils import load_ordered_yaml
32class MetricSet(JsonSerializationMixin):
33 r"""A collection of `Metric`\ s.
35 Parameters
36 ----------
37 metrics : sequence of `Metric` instances, optional
38 `Metric`\ s to be contained within the ``MetricSet``.
39 """
41 def __init__(self, metrics=None):
42 # Internal dict of Metrics. The MetricSet manages access through its
43 # own mapping API.
44 self._metrics = {}
46 if metrics is not None:
47 for metric in metrics:
48 if not isinstance(metric, Metric): 48 ↛ 49line 48 didn't jump to line 49 because the condition on line 48 was never true
49 message = '{0!r} is not a Metric-type'.format(metric)
50 raise TypeError(message)
51 self._metrics[metric.name] = metric
53 @classmethod
54 def load_metrics_package(cls, package_name_or_path='verify_metrics',
55 subset=None):
56 """Create a MetricSet from a Verification Framework metrics package.
58 Parameters
59 ----------
60 package_name_or_path : `str`, optional
61 Name of an EUPS package that hosts metric and specification
62 definition YAML files **or** the file path to a metrics package.
63 ``'verify_metrics'`` is the default package, and is where metrics
64 and specifications are defined for most packages.
65 subset : `str`, optional
66 If set, only metrics for this package are loaded. For example, if
67 ``subset='validate_drp'``, only ``validate_drp`` metrics are
68 included in the `MetricSet`. This argument is equivalent to the
69 `MetricSet.subset` method. Default is `None`.
71 Returns
72 -------
73 metric_set : `MetricSet`
74 A `MetricSet` containing `Metric` instances.
76 See also
77 --------
78 lsst.verify.MetricSet.load_single_package
80 Notes
81 -----
82 EUPS packages that host metrics and specification definitions for the
83 Verification Framework have top-level directories named ``'metrics'``
84 and ``'specs'``. The metrics package chosen with the
85 ``package_name_or_path`` argument. The default metric package for
86 LSST Science Pipelines is ``verify_metrics``.
88 To make a `MetricSet` from a single package's YAML metric definition
89 file that **is not** contained in a metrics package,
90 use `load_single_package` instead.
91 """
92 # A bare name (no "/") may be an EUPS package, expressed as an
93 # ``eups://`` URI so that resolution is handled by lsst.resources
94 # without a direct EUPS lookup here. Anything containing a "/" is a
95 # filesystem path or URI; such values would produce an empty netloc
96 # (and a ValueError) if forced into an ``eups://`` URI, so skip the
97 # EUPS attempt for them.
98 package_dir = None
99 if "/" not in package_name_or_path:
100 eups_dir = ResourcePath(f"eups://{package_name_or_path}/",
101 forceDirectory=True)
102 if eups_dir.exists():
103 package_dir = eups_dir
104 if package_dir is None:
105 # Fall back to a filesystem path (or URI) instead.
106 package_dir = ResourcePath(package_name_or_path,
107 forceDirectory=True,
108 forceAbsolute=True)
110 metrics_dirname = package_dir.join('metrics', forceDirectory=True)
111 if not metrics_dirname.exists():
112 message = 'Metrics directory {0} not found'
113 raise OSError(message.format(metrics_dirname))
115 metrics = []
117 if subset is not None:
118 # Load only a single package's YAML file
119 metrics_yaml_paths = [metrics_dirname.join(f"{subset}.yaml")]
120 else:
121 # Load all package's YAML files (top level only)
122 _, _, filenames = next(
123 metrics_dirname.walk(file_filter=r'.*\.yaml$'))
124 metrics_yaml_paths = [metrics_dirname.join(name)
125 for name in filenames]
127 for metrics_yaml_path in metrics_yaml_paths:
128 new_metrics = MetricSet._load_metrics_yaml(metrics_yaml_path)
129 metrics.extend(new_metrics)
131 return cls(metrics)
133 @classmethod
134 def load_single_package(cls, metrics_yaml_path):
135 """Create a MetricSet from a single YAML file containing metric
136 definitions for a single package.
138 Returns
139 -------
140 metric_set : `MetricSet`
141 A `MetricSet` containing `Metric` instances found in the YAML
142 file.
144 See also
145 --------
146 lsst.verify.MetricSet.load_metrics_package
148 Notes
149 -----
150 The YAML file's name, without extension, is taken as the package
151 name for all metrics.
153 For example, ``validate_drp.yaml`` contains metrics that are
154 identified as belonging to the ``validate_drp`` package.
155 """
156 metrics = MetricSet._load_metrics_yaml(metrics_yaml_path)
157 return cls(metrics)
159 @staticmethod
160 def _load_metrics_yaml(metrics_yaml_path):
161 # package name is inferred from YAML file name (by definition)
162 metrics_yaml_path = ResourcePath(metrics_yaml_path, forceAbsolute=True)
163 package_name = metrics_yaml_path.updatedExtension('').basename()
165 metrics = []
166 with metrics_yaml_path.open() as f:
167 yaml_doc = load_ordered_yaml(f)
168 for metric_name, metric_doc in yaml_doc.items():
169 name = Name(package=package_name, metric=metric_name)
170 # throw away a 'name' field if there happens to be one
171 metric_doc.pop('name', None)
172 # Create metric instance
173 metric = Metric.deserialize(name=name, **metric_doc)
174 metrics.append(metric)
175 return metrics
177 @classmethod
178 def deserialize(cls, metrics=None):
179 """Deserialize metric JSON objects into a MetricSet instance.
181 Parameters
182 ----------
183 metrics : `list`
184 List of metric JSON serializations (typically created by
185 `MetricSet.json`).
187 Returns
188 -------
189 metric_set : `MetricSet`
190 `MetricSet` instance.
191 """
192 instance = cls()
193 for metric_doc in metrics:
194 metric = Metric.deserialize(**metric_doc)
195 instance.insert(metric)
196 return instance
198 @property
199 def json(self):
200 """A JSON-serializable object (`list`)."""
201 doc = JsonSerializationMixin._jsonify_list(
202 [metric for name, metric in self.items()]
203 )
204 return doc
206 def __getitem__(self, key):
207 if not isinstance(key, Name):
208 key = Name(metric=key)
209 return self._metrics[key]
211 def __setitem__(self, key, value):
212 if not isinstance(key, Name):
213 key = Name(metric=key)
215 # Key name must be for a metric
216 if not key.is_metric: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true
217 message = 'Key {0!r} is not a metric name'.format(key)
218 raise KeyError(message)
220 # value must be a metric type
221 if not isinstance(value, Metric): 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true
222 message = 'Expected {0!s}={1!r} to be a Metric-type'.format(
223 key, value)
224 raise TypeError(message)
226 # Metric name and key name must be consistent
227 if value.name != key:
228 message = 'Key {0!s} inconsistent with Metric {0!s}'
229 raise KeyError(message.format(key, value))
231 self._metrics[key] = value
233 def __delitem__(self, key):
234 if not isinstance(key, Name): 234 ↛ 236line 234 didn't jump to line 236 because the condition on line 234 was always true
235 key = Name(metric=key)
236 del self._metrics[key]
238 def __len__(self):
239 return len(self._metrics)
241 def __contains__(self, key):
242 if not isinstance(key, Name):
243 key = Name(metric=key)
244 return key in self._metrics
246 def __iter__(self):
247 for key in self._metrics:
248 yield key
250 def __str__(self):
251 count = len(self)
252 if count == 0:
253 count_str = 'empty'
254 elif count == 1:
255 count_str = '1 Metric'
256 else:
257 count_str = '{count:d} Metrics'.format(count=count)
258 return '<MetricSet: {0}>'.format(count_str)
260 def __eq__(self, other):
261 if len(self) != len(other): 261 ↛ 262line 261 didn't jump to line 262 because the condition on line 261 was never true
262 return False
264 for name, metric in self.items():
265 try:
266 if metric != other[name]: 266 ↛ 267line 266 didn't jump to line 267 because the condition on line 266 was never true
267 return False
268 except KeyError:
269 return False
271 return True
273 def __ne__(self, other):
274 return not self.__eq__(other)
276 def __iadd__(self, other):
277 """Merge another `MetricSet` into this one.
279 Parameters
280 ---------
281 other : `MetricSet`
282 Another `MetricSet`. Metrics in ``other`` that do exist in this
283 set are added to this one. Metrics in ``other`` replace metrics of
284 the same name in this one.
286 Returns
287 -------
288 self : `MetricSet`
289 This `MetricSet`.
291 Notes
292 -----
293 Equivalent to `update`.
294 """
295 self.update(other)
296 return self
298 def insert(self, metric):
299 """Insert a `Metric` into the set.
301 Any pre-existing metric with the same name is replaced
303 Parameters
304 ----------
305 metric : `Metric`
306 A metric.
307 """
308 self[metric.name] = metric
310 def keys(self):
311 r"""Get a list of metric names included in the set
313 Returns
314 -------
315 keys : `list` of `Name`
316 List of `Name`\ s included in the set.
317 """
318 return self._metrics.keys()
320 def items(self):
321 """Iterate over ``(name, metric)`` pairs in the set.
323 Yields
324 ------
325 item : tuple
326 Tuple containing:
328 - `Name` of the `Metric`
329 - `Metric` instance
330 """
331 for item in self._metrics.items():
332 yield item
334 def subset(self, package=None, tags=None):
335 """Create a new `MetricSet` with metrics belonging to a single
336 package and/or tag.
338 Parameters
339 ----------
340 package : `str` or `lsst.verify.Name`, optional
341 Name of the package to subset metrics by. If the package name
342 is ``'pkg_a'``, then metric ``'pkg_a.metric_1'`` would be
343 **included** in the subset, while ``'pkg_b.metric_2'`` would be
344 **excluded**.
345 tags : sequence of `str`, optional
346 Tags to select metrics by. These tags must be a subset (``<=``)
347 of the `Metric.tags` for the metric to be selected.
349 Returns
350 -------
351 metric_subset : `MetricSet`
352 Subset of this metric set containing only metrics belonging
353 to the specified package and/or tag.
355 Notes
356 -----
357 If both ``package`` and ``tag`` are provided then the resulting
358 `MetricSet` contains the **intersection** of the package-based and
359 tag-based selections. That is, metrics will belong to ``package``
360 and posess the tag ``tag``.
361 """
362 if package is not None and not isinstance(package, Name):
363 package = Name(package=package)
365 if tags is not None:
366 tags = set(tags)
368 if package is not None and tags is None:
369 metrics = [metric for metric_name, metric in self._metrics.items()
370 if metric_name in package]
372 elif package is not None and tags is not None:
373 metrics = [metric for metric_name, metric in self._metrics.items()
374 if metric_name in package
375 if tags <= metric.tags]
377 elif package is None and tags is not None: 377 ↛ 382line 377 didn't jump to line 382 because the condition on line 377 was always true
378 metrics = [metric for metric_name, metric in self._metrics.items()
379 if tags <= metric.tags]
381 else:
382 metrics = []
384 return MetricSet(metrics)
386 def update(self, other):
387 """Merge another `MetricSet` into this one.
389 Parameters
390 ----------
391 other : `MetricSet`
392 Another `MetricSet`. Metrics in ``other`` that do exist in this
393 set are added to this one. Metrics in ``other`` replace metrics of
394 the same name in this one.
395 """
396 for _, metric in other.items():
397 self.insert(metric)
399 def _repr_html_(self):
400 """Make an HTML representation of metrics for Jupyter notebooks.
401 """
402 name_col = []
403 tags_col = []
404 units_col = []
405 description_col = []
406 reference_col = []
408 metric_names = list(self.keys())
409 metric_names.sort()
411 for metric_name in metric_names:
412 metric = self[metric_name]
414 name_col.append(str(metric_name))
416 tags = list(metric.tags)
417 tags.sort()
418 tags_col.append(', '.join(tags))
420 units_col.append("{0:latex}".format(metric.unit))
422 description_col.append(metric.description)
424 reference_col.append(metric.reference)
426 table = Table([name_col, description_col, units_col, reference_col,
427 tags_col],
428 names=['Name', 'Description', 'Units', 'Reference',
429 'Tags'])
430 return table._repr_html_()