Coverage for python/lsst/verify/specset.py: 12%
354 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-19 02:05 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-19 02:05 -0800
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ['SpecificationSet']
23from collections import OrderedDict
24import copy
25import os
26import re
28from astropy.table import Table
30from lsst.utils import getPackageDir
32from .errors import SpecificationResolutionError
33from .jsonmixin import JsonSerializationMixin
34from .naming import Name
35from .spec.base import Specification
36from .spec.threshold import ThresholdSpecification
37from .yamlutils import merge_documents, load_all_ordered_yaml
38from .report import Report
41# Pattern for SpecificationPartial names
42# package:path#name
43PARTIAL_PATTERN = re.compile(r'^(?:(?P<package>\S+):)'
44 r'?(?P<path>\S+)?#(?P<name>\S+)$')
47class SpecificationSet(JsonSerializationMixin):
48 r"""A collection of `Specification`\ s.
50 Parameters
51 ----------
52 specifications : `list` or `tuple` of `Specification` instances
53 A sequence of `Specification`-type instances.
54 partials : `list` or `tuple` of `SpecificationPartial` instances
55 A sequence of `SpecificationPartial` instances. These partials
56 can be used as bases for specification definitions.
57 """
59 def __init__(self, specifications=None, partials=None):
60 # Specifications, keyed by Name (a specification name)
61 self._specs = {}
63 # SpecificationPartial instances, keyed by the fully-qualified
64 # name: ``package_name:yaml_id#name``.
65 self._partials = {}
67 if specifications is not None:
68 for spec in specifications:
69 if not isinstance(spec, Specification):
70 message = '{0!r} must be a Specification type'
71 raise TypeError(message.format(spec))
73 self._specs[spec.name] = spec
75 if partials is not None:
76 for partial in partials:
77 if not isinstance(partial, SpecificationPartial):
78 message = '{0!r} must be a SpecificationPartial type'
79 raise TypeError(message.format(partial))
81 self._partials[partial.name] = partial
83 @classmethod
84 def deserialize(cls, specifications=None):
85 """Deserialize a specification set from a JSON serialization.
87 Parameters
88 ----------
89 specifications : `list`, optional
90 List of specification JSON objects.
92 Returns
93 -------
94 spec_set : `SpecificationSet`
95 `SpecificationSet` instance.
96 """
97 instance = cls()
99 if specifications is not None:
100 for spec_doc in specifications:
101 # FIXME DM-8477 Need a registry to support multiple types
102 # check type
103 if 'threshold' in spec_doc:
104 spec = ThresholdSpecification.deserialize(**spec_doc)
105 else:
106 message = ("We only support threshold-type "
107 "specifications\n"
108 "{0!r}".format(spec_doc))
109 raise NotImplementedError(message)
110 instance.insert(spec)
112 return instance
114 @classmethod
115 def load_metrics_package(cls, package_name_or_path='verify_metrics',
116 subset=None):
117 """Create a `SpecificationSet` from an Verification Framework metrics
118 package.
120 Parameters
121 ----------
122 package_name_or_path : `str`, optional
123 Name of an EUPS package that hosts metric and specification
124 definition YAML files **or** the file path to a metrics package.
125 ``verify_metrics`` is the default package, and is where metrics
126 and specifications are defined for most packages.
127 subset : `str`, optional
128 If set, only specifications defined for this package are loaded.
129 For example, if ``subset='validate_drp'``, only ``validate_drp``
130 specifications are included in the SpecificationSet. This argument
131 is equivalent to the `SpecificationSet.subset` method. Default
132 is `None`.
134 Returns
135 -------
136 spec_set : `SpecificationSet`
137 A `SpecificationSet` containing `Specification` instances.
139 See also
140 --------
141 lsst.verify.SpecificationSet.load_single_package
143 Notes
144 -----
145 EUPS packages that host metrics and specification definitions for the
146 Verification Framework have top-level directories named ``'metrics'``
147 and ``'specs'``.
149 Within ``'specs/'``, directories are named after *packages* that
150 have defined metrics. Contained within these directories are YAML files
151 defining specifications for those metrics.
153 To make a `SpecificationSet` from a single package's YAML definition
154 directory that **is not** contained in a metrics package, use
155 `load_single_package` instead.
156 """
157 try:
158 # Try an EUPS package name
159 package_dir = getPackageDir(package_name_or_path)
160 except LookupError:
161 # Try as a filesystem path instead
162 package_dir = package_name_or_path
163 finally:
164 package_dir = os.path.abspath(package_dir)
166 specs_dirname = os.path.join(package_dir, 'specs')
167 if not os.path.isdir(specs_dirname):
168 message = 'Specifications directory {0} not found'
169 raise OSError(message.format(specs_dirname))
171 instance = cls()
173 if subset is not None:
174 # Load specifications only for the package given by `subset`
175 package_names = [subset]
176 else:
177 # Load specifications for each 'package' within specs/
178 package_names = os.listdir(specs_dirname)
180 for name in package_names:
181 package_specs_dirname = os.path.join(specs_dirname, name)
182 if not os.path.isdir(package_specs_dirname):
183 continue
184 instance._load_package_dir(package_specs_dirname)
186 return instance
188 @classmethod
189 def load_single_package(cls, package_specs_dirname):
190 """Create a `SpecificationSet` from a filesystem directory containing
191 specification YAML files for a single package.
193 Parameters
194 ----------
195 package_specs_dirname : `str`
196 Directory containing specification definition YAML files for
197 metrics of a single package. The name of this directory (final
198 path component) is taken as the name of the package.
200 Returns
201 -------
202 spec_set : `SpecificationSet`
203 A `SpecificationSet` containing `Specification` instances.
205 See also
206 --------
207 lsst.verify.SpecificationSet.load_metrics_package
209 Notes
210 -----
211 This SpecificationSet constructor is useful for loading specifications
212 from a directory containing specification definitions for a single
213 package. The directory name is interpreted as a package name
214 for fully-qualified metric and specification names.
216 To load a Verification Framework metrics package, like
217 ``verify_metrics``, with specifications for multple packages,
218 use `load_metrics_packge` instead.
219 """
220 instance = cls()
221 instance._load_package_dir(package_specs_dirname)
223 return instance
225 def _load_package_dir(self, package_specs_dirname):
226 yaml_extensions = ('.yaml', '.yml')
227 package_specs_dirname = os.path.abspath(package_specs_dirname)
229 all_docs = []
231 for (root_dir, _, filenames) in os.walk(package_specs_dirname):
232 for filename in filenames:
233 if os.path.splitext(filename)[-1] not in yaml_extensions:
234 continue
235 filename = os.path.join(root_dir, filename)
236 spec_docs, partial_docs = SpecificationSet._load_yaml_file(
237 filename,
238 package_specs_dirname)
239 all_docs.extend(partial_docs)
240 all_docs.extend(spec_docs)
242 # resolve inheritance and Specification* instances when possible
243 while len(all_docs) > 0:
244 redo_queue = []
246 for doc in all_docs:
247 try:
248 doc = self.resolve_document(doc)
249 except SpecificationResolutionError:
250 # try again later
251 redo_queue.append(doc)
252 continue
254 if 'id' in doc:
255 partial = SpecificationPartial(doc)
256 self._partials[partial.name] = partial
257 else:
258 # Make sure the name is fully qualified
259 # since _process_specification_yaml_doc may not have
260 # finished this yet
261 doc['name'] = SpecificationSet._normalize_spec_name(
262 doc['name'], metric=doc.get('metric', None),
263 package=doc.get('package', None))
265 # FIXME DM-8477 Need a registry to support multiple types
266 if 'threshold' not in doc:
267 message = ("We only support threshold-type "
268 "specifications\n"
269 "{0!r}".format(doc))
270 raise NotImplementedError(message)
271 spec = ThresholdSpecification.deserialize(**doc)
273 name = spec.name
275 if not name.is_fq:
276 message = (
277 'Fully-qualified name not resolved for'
278 '{0!s}'.format(spec))
279 raise SpecificationResolutionError(message)
281 self._specs[name] = spec
283 if len(redo_queue) == len(all_docs):
284 message = ("There are unresolved specification "
285 "documents: {0!r}")
286 raise SpecificationResolutionError(message.format(redo_queue))
288 all_docs = redo_queue
290 @staticmethod
291 def _load_yaml_file(yaml_file_path, package_dirname):
292 r"""Ingest specifications and partials from a single YAML file.
294 Parameters
295 ----------
296 yaml_file_path : `str`
297 File path of the specification YAML file.
298 package_dirname : `str`
299 Path of the root directory for a package's specifications.
301 Returns
302 -------
303 spec_docs : `list`
304 Specification YAML documents (`~collections.OrderedDict`\ s).
305 partial_docs : `list`
306 Specificaton partial YAML documents
307 (`~collections.OrderedDict`\ s).
309 Notes
310 -----
311 As it loads specification and specification partial documents from
312 YAML, it normalizes and enriches the documents with context necessary
313 for constructing Specification and SpecificationPartial instances
314 in other methods:
316 - A ``'package`` field is added.
317 - A ``'metric'`` field is added, if possible.
318 - Specification names are made fully-qualified with the
319 format ``package.metric.spec_name`` if possible (as `str`).
320 - Partial IDs are fully-qualified with the format
321 ``package:relative_yaml_path_without_extension#id``, for example
322 ``validate_drp:custom/gri#base``.
323 - The ``base`` field is processed so that each partial or specification
324 name is fully-qualified.
325 """
326 # Ensure paths are absolute so we can make relative paths and
327 # determine the package name from the last directory component of
328 # the package_dirname.
329 package_dirname = os.path.abspath(package_dirname)
330 yaml_file_path = os.path.abspath(yaml_file_path)
332 if not os.path.isdir(package_dirname):
333 message = 'Specification package directory {0!r} not found.'
334 raise OSError(message.format(package_dirname))
335 if not os.path.isfile(yaml_file_path):
336 message = 'Specification YAML file {0!r} not found.'
337 raise OSError(message.format(yaml_file_path))
339 # Name of the stack package these specifcation belong to, based
340 # on our metrics/specification package directory structure.
341 package_name = package_dirname.split(os.path.sep)[-1]
343 # path identifier used in names for partials does not have an
344 # extension, and must have '/' directory separators.
345 yaml_id = os.path.relpath(yaml_file_path,
346 start=package_dirname)
347 yaml_id = os.path.splitext(yaml_id)[0]
348 yaml_id = '/'.join(yaml_id.split(os.path.sep))
350 spec_docs = []
351 partial_docs = []
352 with open(yaml_file_path) as stream:
353 parsed_docs = load_all_ordered_yaml(stream)
355 for doc in parsed_docs:
356 doc['package'] = package_name
358 if 'id' in doc:
359 # Must be a partial
360 doc = SpecificationSet._process_partial_yaml_doc(
361 doc, yaml_id)
362 partial_docs.append(doc)
364 else:
365 # Must be a specification
366 doc = SpecificationSet._process_specification_yaml_doc(
367 doc, yaml_id)
368 spec_docs.append(doc)
370 return spec_docs, partial_docs
372 @staticmethod
373 def _process_specification_yaml_doc(doc, yaml_id):
374 """Process a specification yaml document.
376 Principle functionality is:
378 1. Make ``name`` fully qualified (if possible).
379 2. Add ``metric`` field (if possible).
380 3. Add ``package`` field (if possible).
381 """
382 # Ensure name is fully specified
383 metric = doc.get('metric', None)
384 package = doc.get('package', None)
386 try:
387 doc['name'] = SpecificationSet._normalize_spec_name(
388 doc['name'], metric=metric, package=package)
390 _name = Name(doc['name'])
391 doc['metric'] = _name.metric
392 doc['package'] - _name.package
393 except TypeError:
394 # Can't resolve the fully-qualified specification
395 # name until inheritance is resolved. No big deal.
396 pass
398 # Make all bases fully-specified
399 if 'base' in doc:
400 processed_bases = SpecificationSet._process_bases(
401 doc['base'], doc['package'], yaml_id)
402 doc['base'] = processed_bases
404 return doc
406 @staticmethod
407 def _process_partial_yaml_doc(doc, yaml_id):
408 """Process a specification yaml document.
410 Principle functionality is:
412 1. Make `id` fully specified.
413 2. Make bases fully specified.
414 """
415 package = doc['package']
417 # Ensure the id is fully specified
418 doc['id'] = SpecificationSet._normalize_partial_name(
419 doc['id'],
420 current_yaml_id=yaml_id,
421 package=package)
423 # Make all bases fully-specified
424 if 'base' in doc:
425 processed_bases = SpecificationSet._process_bases(
426 doc['base'], doc['package'], yaml_id)
427 doc['base'] = processed_bases
429 return doc
431 @staticmethod
432 def _process_bases(bases, package_name, yaml_id):
433 if not isinstance(bases, list):
434 bases = [bases]
436 processed_bases = []
437 for base_name in bases:
438 if '#' in base_name:
439 # Base name points is a partial
440 base_name = SpecificationSet._normalize_partial_name(
441 base_name,
442 current_yaml_id=yaml_id,
443 package=package_name)
444 else:
445 # Base name points to a specification
446 base_name = SpecificationSet._normalize_spec_name(
447 base_name,
448 package=package_name)
450 processed_bases.append(base_name)
452 return processed_bases
454 @staticmethod
455 def _normalize_partial_name(name, current_yaml_id=None, package=None):
456 """Normalize a partial's identifier.
458 >>> SpecificationSet._normalize_partial_name(
459 ... '#base',
460 ... current_yaml_id='custom/bases',
461 ... package='validate_drp')
462 'validate_drp:custom/bases#base'
463 """
464 if '#' not in name:
465 # Name is probably coming from a partial's own `id` field
466 # which just has the post-# part of a specification's fully
467 # qualified name.
468 name = '#' + name
470 matches = PARTIAL_PATTERN.search(name)
472 # Use info from user arguments if not given directly.
473 # Thus a user can't override info already in the name
474 _package = matches.group('package')
475 if _package is None:
476 _package = package
477 _path = matches.group('path')
478 if _path is None:
479 _path = current_yaml_id
480 partial_name = matches.group('name')
482 # Create the fully-specified name
483 fmt = '{package}:{path}#{name}'
484 return fmt.format(package=_package,
485 path=_path,
486 name=partial_name)
488 @staticmethod
489 def _normalize_spec_name(name, metric=None, package=None):
490 """Normalize a specification name to a fully-qualified specification
491 name.
493 >>> SpecificationSet._normalize_spec_name('PA1.design',
494 ... package='validate_drp')
495 'validate_drp.PA1.design'
496 """
497 name = Name(package=package, metric=metric, spec=name)
498 return name.fqn
500 @property
501 def json(self):
502 doc = JsonSerializationMixin._jsonify_list(
503 [spec for name, spec in self.items()]
504 )
505 return doc
507 def __str__(self):
508 count = len(self)
509 if count == 0:
510 count_str = 'empty'
511 elif count == 1:
512 count_str = '1 Specification'
513 else:
514 count_str = '{count:d} Specifications'.format(count=count)
515 return '<SpecificationSet: {0}>'.format(count_str)
517 def __len__(self):
518 """Number of `Specifications` in the set."""
519 return len(self._specs)
521 def __contains__(self, name):
522 """Check if the set contains a `Specification` by name."""
523 if isinstance(name, str) and '#' in name:
524 # must be a partial's name
525 return name in self._partials
527 else:
528 # must be a specification.
529 if not isinstance(name, Name):
530 name = Name(spec=name)
532 return name in self._specs
534 def __getitem__(self, name):
535 """Retrive a Specification or a SpecificationPartial."""
536 if isinstance(name, str) and '#' in name:
537 # must be a partial's name
538 return self._partials[name]
540 else:
541 # must be a specification.
542 if not isinstance(name, Name):
543 name = Name(spec=name)
545 if not name.is_spec:
546 message = 'Expected key {0!r} to resolve a specification'
547 raise KeyError(message.format(name))
549 return self._specs[name]
551 def __setitem__(self, key, value):
552 if isinstance(key, str) and '#' in key:
553 # must be a partial's name
554 if not isinstance(value, SpecificationPartial):
555 message = ('Expected {0!s}={1!r} to be a '
556 'SpecificationPartial-type')
557 raise TypeError(message.format(key, value))
559 # Ensure key and value.name are consistent
560 if key != value.name:
561 message = ("Key {0!s} does not match the "
562 "SpecificationPartial's name {1!s})")
563 raise KeyError(message.format(key, value.name))
564 self._partials[key] = value
566 else:
567 # must be a specification.
568 if not isinstance(key, Name):
569 key = Name(spec=key)
571 if not key.is_spec:
572 message = 'Expected key {0!r} to resolve a specification'
573 raise KeyError(message.format(key))
575 if not isinstance(value, Specification):
576 message = ('Expected {0!s}={1!r} to be a '
577 'Specification-type')
578 raise TypeError(message.format(key, value))
580 # Ensure key and value.name are consistent
581 if key != value.name:
582 message = ("Key {0!s} does not match the "
583 "Specification's name {1!s})")
584 raise KeyError(message.format(key, value.name))
586 self._specs[key] = value
588 def __delitem__(self, key):
589 if isinstance(key, str) and '#' in key:
590 # must be a partial's name
591 del self._partials[key]
593 else:
594 # must be a specification
595 if not isinstance(key, Name):
596 key = Name(spec=key)
598 del self._specs[key]
600 def __iter__(self):
601 for key in self._specs:
602 yield key
604 def __eq__(self, other):
605 if len(self) != len(other):
606 return False
608 for name, spec in self.items():
609 try:
610 if spec != other[name]:
611 return False
612 except KeyError:
613 return False
615 return True
617 def __ne__(self, other):
618 return not self.__eq__(other)
620 def __iadd__(self, other):
621 """Merge another `SpecificationSet` into this one.
623 Parameters
624 ---------
625 other : `SpecificationSet`
626 Another `SpecificationSet`. Specification in ``other`` that do
627 exist in this set are added to this one. Specification in ``other``
628 replace specifications of the same name in this one.
630 Returns
631 -------
632 self : `SpecificationSet`
633 This `SpecificationSet`.
635 Notes
636 -----
637 Equivalent to `update`.
638 """
639 self.update(other)
640 return self
642 def keys(self):
643 """Get a sequence of specification names, which are keys to the set.
645 Returns
646 -------
647 keys : sequence of `Name`
648 Keys to the specification set.
649 """
650 return self._specs.keys()
652 def items(self):
653 """Iterate over name, specification pairs.
655 Yields
656 ------
657 item : `tuple`
658 Tuple containing:
660 - `Name` of the specification.
661 - `Specification`-type object.
662 """
663 for name, spec in self._specs.items():
664 yield name, spec
666 def insert(self, spec):
667 """Insert a `Specification` into the set.
669 A pre-existing specification with the same name is replaced.
671 Parameters
672 ----------
673 spec : `Specification`-type
674 A specification.
675 """
676 key = spec.name
677 self[key] = spec
679 def update(self, other):
680 r"""Merge another `SpecificationSet` into this one.
682 Parameters
683 ----------
684 other : `SpecificationSet`
685 Another `SpecificationSet`. `Specification`\ s in ``other`` that do
686 not exist in this set are added to this one. `Specification`\ s in
687 ``other`` replace specifications of the same name in this one.
688 """
689 for _, spec in other.items():
690 self.insert(spec)
692 def resolve_document(self, spec_doc):
693 """Resolve inherited properties in a specification document using
694 specifications available in the repo.
696 Parameters
697 ----------
698 spec_doc : `dict`
699 A specification document. A document is typically either a YAML
700 document, where the specification is defined, or a JSON object
701 that was serialized from a `~lsst.validate.base.Specification`
702 instance.
704 Returns
705 -------
706 spec_doc : `OrderedDict`
707 The specification document is returned with bases resolved.
709 Raises
710 ------
711 SpecificationResolutionError
712 Raised when a document's bases cannot be resolved (an inherited
713 `~lsst.validate.base.Specification` cannot be found in the repo).
714 """
715 # Create a copy of the spec_doc so that if the resolution is aborted
716 # we haven't modified the original document
717 spec_doc = copy.deepcopy(spec_doc)
719 # Goal is to process all specifications and partials mentioned in
720 # the 'base' field (first in, first out) and merge their information
721 # to the spec_doc.
722 if 'base' in spec_doc:
723 # Coerce 'base' field into a list for consistency
724 if isinstance(spec_doc['base'], str):
725 spec_doc['base'] = [spec_doc['base']]
727 built_doc = OrderedDict()
729 # Process all base dependencies into the specification
730 # document until all are merged
731 while len(spec_doc['base']) > 0:
732 # Select first base (first in, first out queue)
733 base_name = spec_doc['base'][0]
735 # Get the base: it's either another specification or a partial
736 if '#' in base_name:
737 # We make base names fully qualifed when loading them
738 try:
739 base_spec = self._partials[base_name]
740 except KeyError:
741 # Abort because this base is not available yet
742 raise SpecificationResolutionError
744 else:
745 # Must be a specification.
746 # Resolve its name (use package info from present doc since
747 # they're consistent).
748 base_name = Name(package=spec_doc['package'],
749 spec=base_name)
750 # Try getting the specification from the repo
751 try:
752 base_spec = self[base_name]
753 except KeyError:
754 # Abort because this base is not resolved
755 # or not yet available
756 raise SpecificationResolutionError
758 # Merge this spec_doc onto the base document using
759 # our inheritance algorithm
760 built_doc = merge_documents(built_doc, base_spec.json)
762 # Mix in metric information if available. This is useful
763 # because a specification may only assume its metric
764 # identity from inheritance.
765 try:
766 built_doc['metric'] = base_spec.name.metric
767 except AttributeError:
768 # base spec must be a partial
769 pass
771 # Remove this base spec from the queue
772 del spec_doc['base'][0]
774 # if base list is empty remove it so we don't loop over it again
775 if len(spec_doc['base']) == 0:
776 del spec_doc['base']
778 # Merge this spec_doc onto the base document using
779 # our inheritance algorithm
780 built_doc = merge_documents(built_doc, spec_doc)
782 return built_doc
784 else:
785 # No inheritance to resolve
786 return spec_doc
788 def subset(self, name=None, meta=None, required_meta=None,
789 spec_tags=None, metric_tags=None, metrics=None):
790 """Create a new `SpecificationSet` with specifications belonging to
791 a single package or metric, and that apply to the given metadata.
793 Parameters
794 ----------
795 name : `str` or `lsst.verify.Name`, optional
796 Name to subset specifications by. If this is the name of a package,
797 then all specifications for that package are included in the
798 subset. If this is a metric name, then only specifications
799 for that metric are included in the subset. The metric name
800 must be fully-qualified (that is, it includes a package component).
801 meta : `lsst.verify.Metadata`, optional
802 If supplied, only specifications that apply to the given metadata
803 are included in the subset. Metadata is usually obtained from
804 the `Job.meta` attribute of a `Job` instance. By default,
805 specifications are selected as long as the ``meta`` argument
806 as at least all the terms defined in a specification's metadata
807 query and their term values do not conflict.
808 required_metadata : `dict` or `lsst.verify.Metadata`, optional
809 If supplied, only specifications that have **all** the terms in
810 ``required_metadata`` (and their term values match) are selected.
811 This is opposite to the logic of the ``meta`` argument where a
812 specification with an empty metadata query is always selected,
813 for example. This query is performed with the ``arg_driven=True``
814 mode of `lsst.verify.MetadataQuery`.
815 spec_tags : sequence of `str`, optional
816 A set of specification tag strings. when given, only
817 specifications that have all the given tags are included in the
818 report. For example, ``spec_tags=['LPM-17', 'minimum']``.
819 metric_tags : sequence of `str`, optional
820 A set of metric tag strings. When given, only specifications
821 belonging to metrics that posess **all** given tags are included
822 in the report. For example,
823 ``metric_tags=['LPM-17', 'photometry']`` selects sepifications
824 that have both the ``'LPM-17'`` and ``'photometry'`` tags. If
825 set, also provide a `lsst.verify.MetricSet` with the ``metrics``
826 argument.
827 metrics : `lsst.verify.MetricSet`
828 `~lsst.verify.MetricSet` with metric definitions. This is only
829 needed if a ``metric_tags`` argument is provided.
831 Returns
832 -------
833 spec_subset : `SpecificationSet`
834 Subset of this `SpecificationSet` containing only specifications
835 belonging to the indicated package or metric, and/or that are
836 compatible with the job metadata. Any partials in
837 the SpecificationSet are also included in ``spec_subset``.
839 See also
840 --------
841 lsst.very.MetadataQuery
842 """
843 if metric_tags is not None and metrics is None:
844 message = ('A MetricSet must be provided through the metrics '
845 'argument when subsetting ith metric_tags.')
846 raise ValueError(message)
848 all_partials = [partial
849 for partial_name, partial in self._partials.items()]
851 # Filter by package or metric name
852 if name is not None:
853 if not isinstance(name, Name):
854 name = Name(name)
856 if not name.is_fq:
857 message = '{0!s} is not a fully-qualified name'.format(name)
858 raise RuntimeError(message)
860 specs = [spec for spec_name, spec in self._specs.items()
861 if spec_name in name]
863 spec_subset = SpecificationSet(specifications=specs,
864 partials=all_partials)
865 else:
866 spec_subset = self
868 # Filter by metadata
869 if meta is not None:
870 specs = [spec for spec_name, spec in spec_subset.items()
871 if spec.query_metadata(meta)]
873 spec_subset = SpecificationSet(specifications=specs,
874 partials=all_partials)
876 # Filter by required metadata terms
877 if required_meta is not None:
878 specs = [spec for spec_name, spec in spec_subset.items()
879 if spec.query_metadata(required_meta,
880 arg_driven=True)]
882 spec_subset = SpecificationSet(specifications=specs,
883 partials=all_partials)
885 # Filter by specifiation tags
886 if spec_tags is not None:
887 spec_tags = set(spec_tags)
888 specs = [spec for spec_name, spec in spec_subset.items()
889 if spec_tags <= spec.tags]
891 spec_subset = SpecificationSet(specifications=specs,
892 partials=all_partials)
894 # Filter by metric tags
895 if metric_tags is not None:
896 metric_tags = set(metric_tags)
897 specs = [spec for spec_name, spec in spec_subset.items()
898 if metric_tags <= metrics[spec.metric_name].tags]
900 return spec_subset
902 def report(self, measurements, name=None, meta=None, spec_tags=None,
903 metric_tags=None, metrics=None):
904 """Create a report that details specification tests against the given
905 measurements.
907 Parameters
908 ----------
909 measurements : `lsst.verify.MeasurementSet`
910 Measurements to test.
911 name : `str` or `lsst.verify.Name`, optional
912 A package or metric name to subset specifications by. When set,
913 only measurement and specification combinations belonging to that
914 package or metric are included in the report.
915 meta : `lsst.verifify.Metadata`, optional
916 Job metadata to ensure the specifications are relevant to the
917 measurements. Typically accessed as `Job.meta`.
918 spec_tags : sequence of `str`, optional
919 A set of specification tag strings. when given, only
920 specifications that have all the given tags are included in the
921 report. For example, ``spec_tags=['LPM-17', 'minimum']``.
922 metric_tags : sequence of `str`, optional
923 A set of metric tag strings. When given, only specifications
924 belonging to metrics that posess **all** given tags are included
925 in the report. For example,
926 ``metric_tags=['LPM-17', 'photometry']`` selects sepifications
927 that have both the ``'LPM-17'`` and ``'photometry'`` tags. If
928 set, also provide a `lsst.verify.MetricSet` with the ``metrics``
929 argument.
930 metrics : `lsst.verify.MetricSet`
931 `~lsst.verify.MetricSet` with metric definitions. This is only
932 needed if a ``metric_tags`` argument is provided.
934 Returns
935 -------
936 report : `lsst.verify.Report`
937 Report instance. In a Jupyter notebook, you can view the report
938 by calling `Report.show`.
940 See also
941 --------
942 lsst.verify.Job.report
943 """
944 spec_subset = self.subset(name=name, meta=meta,
945 spec_tags=spec_tags,
946 metric_tags=metric_tags, metrics=metrics)
947 return Report(measurements, spec_subset)
949 def _repr_html_(self):
950 """Make an HTML representation of the SpecificationSet for Jupyter
951 notebooks.
952 """
953 name_col = []
954 tags_col = []
955 test_col = []
957 names = list(self.keys())
958 names.sort()
960 for name in names:
961 spec = self[name]
963 name_col.append(str(name))
965 test_col.append(spec._repr_latex_())
967 tags = list(spec.tags)
968 tags.sort()
969 tags_col.append(', '.join(tags))
971 table = Table([name_col, test_col, tags_col],
972 names=['Name', 'Test', 'Tags'])
973 return table._repr_html_()
976class SpecificationPartial(object):
977 """A specification definition partial, used when parsing specification
978 YAML repositories.
979 """
981 def __init__(self, yaml_doc):
982 self.yaml_doc = yaml_doc
983 self.name = self.yaml_doc.pop('id')
985 def __str__(self):
986 return self.name
988 def __hash__(self):
989 return hash(self.name)
991 @property
992 def json(self):
993 """JSON-serializable representation of the partial."""
994 # This API is for compatibility with Specification classes
995 return self.yaml_doc