Coverage for tests/test_specification_set.py: 27%
215 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 02:05 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 02:05 -0700
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from collections import OrderedDict
23import os
24import unittest
25from io import StringIO
27import astropy.units as u
29from lsst.verify.errors import SpecificationResolutionError
30from lsst.verify.naming import Name
31from lsst.verify.specset import SpecificationSet, SpecificationPartial
32from lsst.verify.spec import ThresholdSpecification
33from lsst.verify.yamlutils import load_ordered_yaml
36class TestSpecificationSet(unittest.TestCase):
37 """Tests for SpecificationSet basic usage."""
39 def setUp(self):
40 self.spec_PA1_design = ThresholdSpecification(
41 'validate_drp.PA1.design', 5. * u.mmag, '<')
42 self.spec_PA1_stretch = ThresholdSpecification(
43 'validate_drp.PA1.stretch', 3. * u.mmag, '<')
44 self.spec_PA2_design = ThresholdSpecification(
45 'validate_drp.PA2_design_gri.srd', 15. * u.mmag, '<=')
47 specs = [self.spec_PA1_design,
48 self.spec_PA1_stretch,
49 self.spec_PA2_design]
51 partial_PA1_doc = OrderedDict([
52 ('id', 'validate_drp.LPM-17-PA1#PA1-Base'),
53 ('threshold', OrderedDict([
54 ('unit', 'mag')
55 ]))
56 ])
57 partial_PA1 = SpecificationPartial(partial_PA1_doc)
59 self.spec_set = SpecificationSet(specifications=specs,
60 partials=[partial_PA1])
62 def test_len(self):
63 self.assertEqual(len(self.spec_set), 3)
65 def test_contains(self):
66 self.assertTrue(self.spec_PA1_design.name in self.spec_set)
67 self.assertTrue('validate_drp.PA1.design' in self.spec_set)
68 self.assertFalse(
69 Name('validate_drp.WeirdMetric.design') in self.spec_set)
71 # Metric, not specification
72 self.assertFalse(
73 'validate_drp.PA1' in self.spec_set)
74 self.assertFalse(
75 Name('validate_drp.PA1') in self.spec_set)
77 def test_getitem(self):
78 # get Specifications when given a specification name
79 self.assertEqual(
80 self.spec_set['validate_drp.PA1.design'],
81 self.spec_PA1_design
82 )
84 # KeyError when requesting a metric (anything not a specification
85 with self.assertRaises(KeyError):
86 self.spec_set['validate_drp.PA1']
88 def test_iter(self):
89 """Test SpecificationSet key iteration."""
90 names = [n for n in self.spec_set]
91 self.assertEqual(len(names), len(self.spec_set))
92 for name in names:
93 self.assertTrue(isinstance(name, Name))
95 def test_iadd(self):
96 """Test SpecifcationSet.__iadd__."""
97 set1 = SpecificationSet([self.spec_PA1_design, self.spec_PA1_stretch])
98 set2 = SpecificationSet([self.spec_PA1_design, self.spec_PA2_design])
100 set1 += set2
102 self.assertIn('validate_drp.PA1.design', set1)
103 self.assertIn('validate_drp.PA1.stretch', set1)
104 self.assertIn('validate_drp.PA2_design_gri.srd', set1)
105 self.assertEqual(len(set1), 3)
107 def test_update(self):
108 """Test SpecificationSet.update()."""
109 set1 = SpecificationSet([self.spec_PA1_design, self.spec_PA1_stretch])
110 set2 = SpecificationSet([self.spec_PA1_design, self.spec_PA2_design])
112 set1.update(set2)
114 self.assertIn('validate_drp.PA1.design', set1)
115 self.assertIn('validate_drp.PA1.stretch', set1)
116 self.assertIn('validate_drp.PA2_design_gri.srd', set1)
117 self.assertEqual(len(set1), 3)
119 def test_resolve_document(self):
120 """Test specification document inheritance resolution."""
121 new_spec_doc = OrderedDict([
122 ('name', 'PA1.relaxed'),
123 ('base', ['PA1.design', 'validate_drp.LPM-17-PA1#PA1-Base']),
124 ('package', 'validate_drp'),
125 ('threshold', OrderedDict([
126 ('value', 1)
127 ]))
128 ])
130 resolved_doc = self.spec_set.resolve_document(new_spec_doc)
132 self.assertEqual(resolved_doc['name'], 'PA1.relaxed')
133 self.assertEqual(resolved_doc['threshold']['unit'], 'mag')
134 self.assertEqual(resolved_doc['threshold']['value'], 1)
135 self.assertEqual(resolved_doc['threshold']['operator'], '<')
136 self.assertNotIn('base', resolved_doc)
138 def test_unresolvable_document(self):
139 """Test that SpecificationResolutionError is raised for unresolveable
140 inheritance bases.
141 """
142 new_spec_doc = OrderedDict([
143 ('name', 'PA1.unresolved'),
144 ('base', 'PA1.non_existent'),
145 ('package', 'validate_drp'),
146 ('threshold', OrderedDict([
147 ('value', 10)
148 ]))
149 ])
151 with self.assertRaises(SpecificationResolutionError):
152 self.spec_set.resolve_document(new_spec_doc)
154 def test_serialization(self):
155 """Test json and deserialize."""
156 json_doc = self.spec_set.json
157 new_spec_set = SpecificationSet.deserialize(json_doc)
158 self.assertEqual(self.spec_set, new_spec_set)
161class TestSpecificationSetGetterSetter(unittest.TestCase):
162 """Test __setitem__, __getitem__ and __delitem__."""
164 def test_mapping(self):
165 spec_PA1_design = ThresholdSpecification(
166 'validate_drp.PA1.design', 5. * u.mmag, '<')
167 spec_PA1_stretch = ThresholdSpecification(
168 'validate_drp.PA1.stretch', 3. * u.mmag, '<')
170 spec_set = SpecificationSet()
171 self.assertEqual(len(spec_set), 0)
173 # This syntax is slightly awkward, which is why we have `insert()` too
174 spec_set[spec_PA1_design.name] = spec_PA1_design
175 self.assertEqual(len(spec_set), 1)
176 self.assertTrue('validate_drp.PA1.design' in spec_set)
178 # Insert
179 spec_set.insert(spec_PA1_stretch)
180 self.assertEqual(len(spec_set), 2)
181 self.assertTrue('validate_drp.PA1.stretch' in spec_set)
183 # Delete
184 del spec_set['validate_drp.PA1.stretch']
185 self.assertEqual(len(spec_set), 1)
186 self.assertFalse('validate_drp.PA1.stretch' in spec_set)
188 # Insert duplicate
189 spec_set[spec_PA1_design.name] = spec_PA1_design
190 self.assertEqual(len(spec_set), 1)
191 self.assertTrue('validate_drp.PA1.design' in spec_set)
193 # Insert weird value
194 with self.assertRaises(TypeError):
195 spec_set['validate_drp.PAX.design'] = 10
197 # __setitem__ insert with a conflicting key.
198 # This is why insert() is preferred.
199 with self.assertRaises(KeyError):
200 spec_set['validate_drp.hello.world'] = spec_PA1_design
203class TestSpecificationSetLoadYamlFile(unittest.TestCase):
204 """Test SpecificationSet._load_yaml_file() and sub-functions."""
206 def setUp(self):
207 self.test_specs_dir = os.path.join(
208 os.path.dirname(__file__),
209 'data/specs')
211 def test_load_yaml_file(self):
212 package = 'validate_drp'
213 package_dirname = os.path.join(self.test_specs_dir, package)
214 path = os.path.join(package_dirname, 'cfht_gri.yaml')
216 spec_docs, partial_docs = SpecificationSet._load_yaml_file(
217 path, package_dirname)
219 self.assertEqual(len(spec_docs), 9)
220 self.assertEqual(len(partial_docs), 1)
222 self.assertEqual(partial_docs[0]['id'], 'validate_drp:cfht_gri#base')
224 def test_process_bases(self):
225 yaml_id = 'dirname/filename'
226 package_name = 'package'
227 bases = ['PA2_minimum_gri.srd', '#base']
228 expected = ['package.PA2_minimum_gri.srd',
229 'package:dirname/filename#base']
230 self.assertEqual(
231 SpecificationSet._process_bases(bases, package_name, yaml_id),
232 expected
233 )
235 def test_process_bases_known_yaml_id(self):
236 """Process bases when a partial already has a yaml path."""
237 yaml_id = 'dirname/filename'
238 package_name = 'package'
239 bases = ['PA2_minimum_gri.srd', 'otherdir/otherfile#base']
240 expected = ['package.PA2_minimum_gri.srd',
241 'package:otherdir/otherfile#base']
242 self.assertEqual(
243 SpecificationSet._process_bases(bases, package_name, yaml_id),
244 expected
245 )
247 def test_normalize_partial_name(self):
248 self.assertEqual(
249 SpecificationSet._normalize_partial_name(
250 'name',
251 current_yaml_id='dirname/filename',
252 package='package'),
253 'package:dirname/filename#name'
254 )
256 self.assertEqual(
257 SpecificationSet._normalize_partial_name(
258 'otherdir/otherfile#name',
259 current_yaml_id='dirname/filename',
260 package='package'),
261 'package:otherdir/otherfile#name'
262 )
264 def test_normalize_spec_name(self):
265 self.assertEqual(
266 SpecificationSet._normalize_spec_name(
267 'metric.spec', package='package'),
268 'package.metric.spec'
269 )
271 # Not resolveable
272 with self.assertRaises(TypeError):
273 SpecificationSet._normalize_spec_name(
274 'spec', package='package'),
276 def test_process_specification_yaml_doc_resolved_name(self):
277 doc = ("name: 'cfht_gri'\n"
278 "package: 'validate_drp'\n"
279 "base: ['PA2_design_gri.srd', '#base']\n")
280 yaml_doc = load_ordered_yaml(StringIO(doc))
281 processed = SpecificationSet._process_specification_yaml_doc(
282 yaml_doc, 'cfht_gri')
284 # name is unresolved
285 self.assertEqual(processed['name'], 'cfht_gri')
286 self.assertEqual(
287 processed['base'],
288 ['validate_drp.PA2_design_gri.srd', 'validate_drp:cfht_gri#base'])
290 def test_process_specification_yaml_doc_unresolved_name(self):
291 doc = ('name: "design_gri"\n'
292 'metric: "PA1"\n'
293 'package: "validate_drp"\n'
294 'base: "#PA1-base"\n'
295 'threshold:\n'
296 ' value: 5.0\n')
297 yaml_doc = load_ordered_yaml(StringIO(doc))
298 processed = SpecificationSet._process_specification_yaml_doc(
299 yaml_doc, 'LPM-17-PA1')
301 # name is resolved
302 self.assertEqual(processed['name'], 'validate_drp.PA1.design_gri')
303 self.assertEqual(
304 processed['base'],
305 ["validate_drp:LPM-17-PA1#PA1-base"])
306 self.assertEqual(
307 processed['metric'],
308 'PA1')
309 self.assertEqual(
310 processed['package'],
311 'validate_drp')
313 def test_process_partial_yaml_doc(self):
314 doc = ("id: 'PA1-base'\n"
315 "metric: 'PA1'\n"
316 "package: 'validate_drp'\n"
317 "threshold:\n"
318 " unit: 'mmag'\n"
319 " operator: '<='\n")
320 yaml_doc = load_ordered_yaml(StringIO(doc))
321 processed = SpecificationSet._process_partial_yaml_doc(
322 yaml_doc, 'LPM-17-PA1')
324 self.assertEqual(
325 processed['id'],
326 'validate_drp:LPM-17-PA1#PA1-base')
327 self.assertEqual(
328 processed['metric'],
329 'PA1')
330 self.assertEqual(
331 processed['package'],
332 'validate_drp')
335class TestSpecificationSetLoadSinglePackage(unittest.TestCase):
336 """Test SpecificationSet.load_single_package."""
338 def setUp(self):
339 self.test_package_dir = os.path.join(
340 os.path.dirname(__file__),
341 'data/specs/validate_drp')
343 def test_load(self):
344 spec_set = SpecificationSet.load_single_package(self.test_package_dir)
345 self.assertTrue('validate_drp.PA1.design_gri' in spec_set)
346 self.assertTrue('validate_drp:cfht_gri#base' in spec_set)
349class TestSpecificationSetLoadMetricsPackage(unittest.TestCase):
350 """Test SpecificationSet.load_metrics_package()."""
352 def setUp(self):
353 # defaults to verify_metrics
354 self.spec_set = SpecificationSet.load_metrics_package()
356 def test_contains(self):
357 self.assertTrue('validate_drp.PA1.design_gri' in self.spec_set)
358 self.assertTrue('validate_drp:cfht_gri/base#cfht' in self.spec_set)
361class TestSpecificationSetNameSubset(unittest.TestCase):
362 """Test creating name-based subsets from a SpecificationSet."""
364 def setUp(self):
365 # defaults to validate_metrics
366 self.spec_set = SpecificationSet.load_metrics_package()
368 def test_validate_drp_subset(self):
369 package = Name('validate_drp')
370 subset = self.spec_set.subset(name='validate_drp')
372 self.assertTrue(isinstance(subset, type(self.spec_set)))
373 self.assertTrue(len(subset) > 0)
375 for spec_name, spec in subset._specs.items():
376 self.assertTrue(spec_name in package)
378 def test_PA1_subset(self):
379 metric = Name('validate_drp.PA1')
380 subset = self.spec_set.subset(name='validate_drp.PA1')
382 self.assertTrue(isinstance(subset, type(self.spec_set)))
383 self.assertTrue(len(subset) > 0)
385 for spec_name, spec in subset._specs.items():
386 self.assertTrue(spec_name in metric)
389class TestSpecificationSetMetadataSubset(unittest.TestCase):
390 """Test creating metadata-based or name and metadata-based subsets
391 from a SpecificationSet.
392 """
394 def setUp(self):
395 s1 = ThresholdSpecification(
396 Name('validate_drp.AM1.design_r'),
397 5. * u.marcsec, '<',
398 metadata_query={'filter_name': 'r'})
399 s2 = ThresholdSpecification(
400 Name('validate_drp.AM1.design_i'),
401 5. * u.marcsec, '<',
402 metadata_query={'filter_name': 'i'})
403 s3 = ThresholdSpecification(
404 Name('validate_drp.AM1.design_HSC_r'),
405 5. * u.marcsec, '<',
406 metadata_query={'filter_name': 'r', 'camera': 'HSC'})
407 s4 = ThresholdSpecification(
408 Name('validate_drp.PA1.design_r'),
409 10 * u.mmag, '<',
410 metadata_query={'filter_name': 'r'})
411 s5 = ThresholdSpecification(
412 Name('validate_drp.AM1.design'),
413 5. * u.marcsec, '<',
414 metadata_query={})
415 self.spec_set = SpecificationSet([s1, s2, s3, s4, s5])
417 def test_metadata_subset(self):
418 """Subset by metadata only."""
419 subset = self.spec_set.subset(meta={'filter_name': 'r'})
421 # In because filter_name: r does not conflict with spec
422 self.assertIn('validate_drp.AM1.design_r', subset)
423 # In because spec has no filter_name metadata query
424 self.assertIn('validate_drp.AM1.design', subset)
425 # Not in because spec has conflicting filter_name
426 self.assertNotIn('validate_drp.AM1.design_i', subset)
427 # Not in because spec requires 'camera' term
428 self.assertNotIn('validate_drp.AM1.design_HSC_r', subset)
429 # In because compatible filter_name
430 self.assertIn('validate_drp.PA1.design_r', subset)
432 def test_name_and_metadata_subset(self):
433 """Subset by name and metadata."""
434 subset = self.spec_set.subset(name='validate_drp.AM1',
435 meta={'filter_name': 'r'})
437 # Spec has right metric, compatible metadata query
438 self.assertIn('validate_drp.AM1.design_r', subset)
439 # Spec has right metric, no metadata query
440 self.assertIn('validate_drp.AM1.design', subset)
441 # Conflicting filter_name
442 self.assertNotIn('validate_drp.AM1.design_i', subset)
443 # Requires camera term
444 self.assertNotIn('validate_drp.AM1.design_HSC_r', subset)
445 # Wrong metric
446 self.assertNotIn('validate_drp.PA1.design_r', subset)
448 def test_required_metadata_subset(self):
449 """Subset by required_meta only."""
450 subset = self.spec_set.subset(required_meta={'filter_name': 'r'})
452 # In because filter_name: r does not conflict with spec
453 self.assertIn('validate_drp.AM1.design_r', subset)
454 # Not in because spec has no filter_name metadata query
455 self.assertNotIn('validate_drp.AM1.design', subset)
456 # Not in because spec has conflicting filter_name
457 self.assertNotIn('validate_drp.AM1.design_i', subset)
458 # In because spec has compatible filter_name
459 self.assertIn('validate_drp.AM1.design_HSC_r', subset)
460 # In because compatible filter_name
461 self.assertIn('validate_drp.PA1.design_r', subset)
463 def test_name_and_required_metadata_subset(self):
464 """Subset by name and required_meta."""
465 subset = self.spec_set.subset(name='validate_drp.AM1',
466 required_meta={'filter_name': 'r'})
468 # In because AM1.design_r has `filter_name: r`
469 self.assertIn('validate_drp.AM1.design_r', subset)
470 # This AM1.design spec doesn't have a `filter_name` term.
471 self.assertNotIn('validate_drp.AM1.design', subset)
472 # Wrong filter_name value
473 self.assertNotIn('validate_drp.AM1.design_i', subset)
474 # Has the filer_name: r term; ignores camera term.
475 self.assertIn('validate_drp.AM1.design_HSC_r', subset)
476 # Not in because PA1 metric
477 self.assertNotIn('validate_drp.PA1.design_r', subset)
479 def test_name_subset(self):
480 """Subset by name."""
481 subset = self.spec_set.subset(name='validate_drp.AM1')
483 # In because AM1 metric
484 self.assertIn('validate_drp.AM1.design_r', subset)
485 self.assertIn('validate_drp.AM1.design_i', subset)
486 self.assertIn('validate_drp.AM1.design_HSC_r', subset)
487 # Not in because PA1 metric
488 self.assertNotIn('validate_drp.PA1.design_HSC_r', subset)
491if __name__ == "__main__": 491 ↛ 492line 491 didn't jump to line 492, because the condition on line 491 was never true
492 unittest.main()