Coverage for tests/test_specification_set.py: 22%

215 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-12 02:15 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from collections import OrderedDict 

23import os 

24import unittest 

25from io import StringIO 

26 

27import astropy.units as u 

28 

29from lsst.verify.errors import SpecificationResolutionError 

30from lsst.verify.naming import Name 

31from lsst.verify.specset import SpecificationSet, SpecificationPartial 

32from lsst.verify.spec import ThresholdSpecification 

33from lsst.verify.yamlutils import load_ordered_yaml 

34 

35 

36class TestSpecificationSet(unittest.TestCase): 

37 """Tests for SpecificationSet basic usage.""" 

38 

39 def setUp(self): 

40 self.spec_PA1_design = ThresholdSpecification( 

41 'validate_drp.PA1.design', 5. * u.mmag, '<') 

42 self.spec_PA1_stretch = ThresholdSpecification( 

43 'validate_drp.PA1.stretch', 3. * u.mmag, '<') 

44 self.spec_PA2_design = ThresholdSpecification( 

45 'validate_drp.PA2_design_gri.srd', 15. * u.mmag, '<=') 

46 

47 specs = [self.spec_PA1_design, 

48 self.spec_PA1_stretch, 

49 self.spec_PA2_design] 

50 

51 partial_PA1_doc = OrderedDict([ 

52 ('id', 'validate_drp.LPM-17-PA1#PA1-Base'), 

53 ('threshold', OrderedDict([ 

54 ('unit', 'mag') 

55 ])) 

56 ]) 

57 partial_PA1 = SpecificationPartial(partial_PA1_doc) 

58 

59 self.spec_set = SpecificationSet(specifications=specs, 

60 partials=[partial_PA1]) 

61 

62 def test_len(self): 

63 self.assertEqual(len(self.spec_set), 3) 

64 

65 def test_contains(self): 

66 self.assertTrue(self.spec_PA1_design.name in self.spec_set) 

67 self.assertTrue('validate_drp.PA1.design' in self.spec_set) 

68 self.assertFalse( 

69 Name('validate_drp.WeirdMetric.design') in self.spec_set) 

70 

71 # Metric, not specification 

72 self.assertFalse( 

73 'validate_drp.PA1' in self.spec_set) 

74 self.assertFalse( 

75 Name('validate_drp.PA1') in self.spec_set) 

76 

77 def test_getitem(self): 

78 # get Specifications when given a specification name 

79 self.assertEqual( 

80 self.spec_set['validate_drp.PA1.design'], 

81 self.spec_PA1_design 

82 ) 

83 

84 # KeyError when requesting a metric (anything not a specification 

85 with self.assertRaises(KeyError): 

86 self.spec_set['validate_drp.PA1'] 

87 

88 def test_iter(self): 

89 """Test SpecificationSet key iteration.""" 

90 names = [n for n in self.spec_set] 

91 self.assertEqual(len(names), len(self.spec_set)) 

92 for name in names: 

93 self.assertTrue(isinstance(name, Name)) 

94 

95 def test_iadd(self): 

96 """Test SpecifcationSet.__iadd__.""" 

97 set1 = SpecificationSet([self.spec_PA1_design, self.spec_PA1_stretch]) 

98 set2 = SpecificationSet([self.spec_PA1_design, self.spec_PA2_design]) 

99 

100 set1 += set2 

101 

102 self.assertIn('validate_drp.PA1.design', set1) 

103 self.assertIn('validate_drp.PA1.stretch', set1) 

104 self.assertIn('validate_drp.PA2_design_gri.srd', set1) 

105 self.assertEqual(len(set1), 3) 

106 

107 def test_update(self): 

108 """Test SpecificationSet.update().""" 

109 set1 = SpecificationSet([self.spec_PA1_design, self.spec_PA1_stretch]) 

110 set2 = SpecificationSet([self.spec_PA1_design, self.spec_PA2_design]) 

111 

112 set1.update(set2) 

113 

114 self.assertIn('validate_drp.PA1.design', set1) 

115 self.assertIn('validate_drp.PA1.stretch', set1) 

116 self.assertIn('validate_drp.PA2_design_gri.srd', set1) 

117 self.assertEqual(len(set1), 3) 

118 

119 def test_resolve_document(self): 

120 """Test specification document inheritance resolution.""" 

121 new_spec_doc = OrderedDict([ 

122 ('name', 'PA1.relaxed'), 

123 ('base', ['PA1.design', 'validate_drp.LPM-17-PA1#PA1-Base']), 

124 ('package', 'validate_drp'), 

125 ('threshold', OrderedDict([ 

126 ('value', 1) 

127 ])) 

128 ]) 

129 

130 resolved_doc = self.spec_set.resolve_document(new_spec_doc) 

131 

132 self.assertEqual(resolved_doc['name'], 'PA1.relaxed') 

133 self.assertEqual(resolved_doc['threshold']['unit'], 'mag') 

134 self.assertEqual(resolved_doc['threshold']['value'], 1) 

135 self.assertEqual(resolved_doc['threshold']['operator'], '<') 

136 self.assertNotIn('base', resolved_doc) 

137 

138 def test_unresolvable_document(self): 

139 """Test that SpecificationResolutionError is raised for unresolveable 

140 inheritance bases. 

141 """ 

142 new_spec_doc = OrderedDict([ 

143 ('name', 'PA1.unresolved'), 

144 ('base', 'PA1.non_existent'), 

145 ('package', 'validate_drp'), 

146 ('threshold', OrderedDict([ 

147 ('value', 10) 

148 ])) 

149 ]) 

150 

151 with self.assertRaises(SpecificationResolutionError): 

152 self.spec_set.resolve_document(new_spec_doc) 

153 

154 def test_serialization(self): 

155 """Test json and deserialize.""" 

156 json_doc = self.spec_set.json 

157 new_spec_set = SpecificationSet.deserialize(json_doc) 

158 self.assertEqual(self.spec_set, new_spec_set) 

159 

160 

161class TestSpecificationSetGetterSetter(unittest.TestCase): 

162 """Test __setitem__, __getitem__ and __delitem__.""" 

163 

164 def test_mapping(self): 

165 spec_PA1_design = ThresholdSpecification( 

166 'validate_drp.PA1.design', 5. * u.mmag, '<') 

167 spec_PA1_stretch = ThresholdSpecification( 

168 'validate_drp.PA1.stretch', 3. * u.mmag, '<') 

169 

170 spec_set = SpecificationSet() 

171 self.assertEqual(len(spec_set), 0) 

172 

173 # This syntax is slightly awkward, which is why we have `insert()` too 

174 spec_set[spec_PA1_design.name] = spec_PA1_design 

175 self.assertEqual(len(spec_set), 1) 

176 self.assertTrue('validate_drp.PA1.design' in spec_set) 

177 

178 # Insert 

179 spec_set.insert(spec_PA1_stretch) 

180 self.assertEqual(len(spec_set), 2) 

181 self.assertTrue('validate_drp.PA1.stretch' in spec_set) 

182 

183 # Delete 

184 del spec_set['validate_drp.PA1.stretch'] 

185 self.assertEqual(len(spec_set), 1) 

186 self.assertFalse('validate_drp.PA1.stretch' in spec_set) 

187 

188 # Insert duplicate 

189 spec_set[spec_PA1_design.name] = spec_PA1_design 

190 self.assertEqual(len(spec_set), 1) 

191 self.assertTrue('validate_drp.PA1.design' in spec_set) 

192 

193 # Insert weird value 

194 with self.assertRaises(TypeError): 

195 spec_set['validate_drp.PAX.design'] = 10 

196 

197 # __setitem__ insert with a conflicting key. 

198 # This is why insert() is preferred. 

199 with self.assertRaises(KeyError): 

200 spec_set['validate_drp.hello.world'] = spec_PA1_design 

201 

202 

203class TestSpecificationSetLoadYamlFile(unittest.TestCase): 

204 """Test SpecificationSet._load_yaml_file() and sub-functions.""" 

205 

206 def setUp(self): 

207 self.test_specs_dir = os.path.join( 

208 os.path.dirname(__file__), 

209 'data/specs') 

210 

211 def test_load_yaml_file(self): 

212 package = 'validate_drp' 

213 package_dirname = os.path.join(self.test_specs_dir, package) 

214 path = os.path.join(package_dirname, 'cfht_gri.yaml') 

215 

216 spec_docs, partial_docs = SpecificationSet._load_yaml_file( 

217 path, package_dirname) 

218 

219 self.assertEqual(len(spec_docs), 9) 

220 self.assertEqual(len(partial_docs), 1) 

221 

222 self.assertEqual(partial_docs[0]['id'], 'validate_drp:cfht_gri#base') 

223 

224 def test_process_bases(self): 

225 yaml_id = 'dirname/filename' 

226 package_name = 'package' 

227 bases = ['PA2_minimum_gri.srd', '#base'] 

228 expected = ['package.PA2_minimum_gri.srd', 

229 'package:dirname/filename#base'] 

230 self.assertEqual( 

231 SpecificationSet._process_bases(bases, package_name, yaml_id), 

232 expected 

233 ) 

234 

235 def test_process_bases_known_yaml_id(self): 

236 """Process bases when a partial already has a yaml path.""" 

237 yaml_id = 'dirname/filename' 

238 package_name = 'package' 

239 bases = ['PA2_minimum_gri.srd', 'otherdir/otherfile#base'] 

240 expected = ['package.PA2_minimum_gri.srd', 

241 'package:otherdir/otherfile#base'] 

242 self.assertEqual( 

243 SpecificationSet._process_bases(bases, package_name, yaml_id), 

244 expected 

245 ) 

246 

247 def test_normalize_partial_name(self): 

248 self.assertEqual( 

249 SpecificationSet._normalize_partial_name( 

250 'name', 

251 current_yaml_id='dirname/filename', 

252 package='package'), 

253 'package:dirname/filename#name' 

254 ) 

255 

256 self.assertEqual( 

257 SpecificationSet._normalize_partial_name( 

258 'otherdir/otherfile#name', 

259 current_yaml_id='dirname/filename', 

260 package='package'), 

261 'package:otherdir/otherfile#name' 

262 ) 

263 

264 def test_normalize_spec_name(self): 

265 self.assertEqual( 

266 SpecificationSet._normalize_spec_name( 

267 'metric.spec', package='package'), 

268 'package.metric.spec' 

269 ) 

270 

271 # Not resolveable 

272 with self.assertRaises(TypeError): 

273 SpecificationSet._normalize_spec_name( 

274 'spec', package='package'), 

275 

276 def test_process_specification_yaml_doc_resolved_name(self): 

277 doc = ("name: 'cfht_gri'\n" 

278 "package: 'validate_drp'\n" 

279 "base: ['PA2_design_gri.srd', '#base']\n") 

280 yaml_doc = load_ordered_yaml(StringIO(doc)) 

281 processed = SpecificationSet._process_specification_yaml_doc( 

282 yaml_doc, 'cfht_gri') 

283 

284 # name is unresolved 

285 self.assertEqual(processed['name'], 'cfht_gri') 

286 self.assertEqual( 

287 processed['base'], 

288 ['validate_drp.PA2_design_gri.srd', 'validate_drp:cfht_gri#base']) 

289 

290 def test_process_specification_yaml_doc_unresolved_name(self): 

291 doc = ('name: "design_gri"\n' 

292 'metric: "PA1"\n' 

293 'package: "validate_drp"\n' 

294 'base: "#PA1-base"\n' 

295 'threshold:\n' 

296 ' value: 5.0\n') 

297 yaml_doc = load_ordered_yaml(StringIO(doc)) 

298 processed = SpecificationSet._process_specification_yaml_doc( 

299 yaml_doc, 'LPM-17-PA1') 

300 

301 # name is resolved 

302 self.assertEqual(processed['name'], 'validate_drp.PA1.design_gri') 

303 self.assertEqual( 

304 processed['base'], 

305 ["validate_drp:LPM-17-PA1#PA1-base"]) 

306 self.assertEqual( 

307 processed['metric'], 

308 'PA1') 

309 self.assertEqual( 

310 processed['package'], 

311 'validate_drp') 

312 

313 def test_process_partial_yaml_doc(self): 

314 doc = ("id: 'PA1-base'\n" 

315 "metric: 'PA1'\n" 

316 "package: 'validate_drp'\n" 

317 "threshold:\n" 

318 " unit: 'mmag'\n" 

319 " operator: '<='\n") 

320 yaml_doc = load_ordered_yaml(StringIO(doc)) 

321 processed = SpecificationSet._process_partial_yaml_doc( 

322 yaml_doc, 'LPM-17-PA1') 

323 

324 self.assertEqual( 

325 processed['id'], 

326 'validate_drp:LPM-17-PA1#PA1-base') 

327 self.assertEqual( 

328 processed['metric'], 

329 'PA1') 

330 self.assertEqual( 

331 processed['package'], 

332 'validate_drp') 

333 

334 

335class TestSpecificationSetLoadSinglePackage(unittest.TestCase): 

336 """Test SpecificationSet.load_single_package.""" 

337 

338 def setUp(self): 

339 self.test_package_dir = os.path.join( 

340 os.path.dirname(__file__), 

341 'data/specs/validate_drp') 

342 

343 def test_load(self): 

344 spec_set = SpecificationSet.load_single_package(self.test_package_dir) 

345 self.assertTrue('validate_drp.PA1.design_gri' in spec_set) 

346 self.assertTrue('validate_drp:cfht_gri#base' in spec_set) 

347 

348 

349class TestSpecificationSetLoadMetricsPackage(unittest.TestCase): 

350 """Test SpecificationSet.load_metrics_package().""" 

351 

352 def setUp(self): 

353 # defaults to verify_metrics 

354 self.spec_set = SpecificationSet.load_metrics_package() 

355 

356 def test_contains(self): 

357 self.assertTrue('validate_drp.PA1.design_gri' in self.spec_set) 

358 self.assertTrue('validate_drp:cfht_gri/base#cfht' in self.spec_set) 

359 

360 

361class TestSpecificationSetNameSubset(unittest.TestCase): 

362 """Test creating name-based subsets from a SpecificationSet.""" 

363 

364 def setUp(self): 

365 # defaults to validate_metrics 

366 self.spec_set = SpecificationSet.load_metrics_package() 

367 

368 def test_validate_drp_subset(self): 

369 package = Name('validate_drp') 

370 subset = self.spec_set.subset(name='validate_drp') 

371 

372 self.assertTrue(isinstance(subset, type(self.spec_set))) 

373 self.assertTrue(len(subset) > 0) 

374 

375 for spec_name, spec in subset._specs.items(): 

376 self.assertTrue(spec_name in package) 

377 

378 def test_PA1_subset(self): 

379 metric = Name('validate_drp.PA1') 

380 subset = self.spec_set.subset(name='validate_drp.PA1') 

381 

382 self.assertTrue(isinstance(subset, type(self.spec_set))) 

383 self.assertTrue(len(subset) > 0) 

384 

385 for spec_name, spec in subset._specs.items(): 

386 self.assertTrue(spec_name in metric) 

387 

388 

389class TestSpecificationSetMetadataSubset(unittest.TestCase): 

390 """Test creating metadata-based or name and metadata-based subsets 

391 from a SpecificationSet. 

392 """ 

393 

394 def setUp(self): 

395 s1 = ThresholdSpecification( 

396 Name('validate_drp.AM1.design_r'), 

397 5. * u.marcsec, '<', 

398 metadata_query={'filter_name': 'r'}) 

399 s2 = ThresholdSpecification( 

400 Name('validate_drp.AM1.design_i'), 

401 5. * u.marcsec, '<', 

402 metadata_query={'filter_name': 'i'}) 

403 s3 = ThresholdSpecification( 

404 Name('validate_drp.AM1.design_HSC_r'), 

405 5. * u.marcsec, '<', 

406 metadata_query={'filter_name': 'r', 'camera': 'HSC'}) 

407 s4 = ThresholdSpecification( 

408 Name('validate_drp.PA1.design_r'), 

409 10 * u.mmag, '<', 

410 metadata_query={'filter_name': 'r'}) 

411 s5 = ThresholdSpecification( 

412 Name('validate_drp.AM1.design'), 

413 5. * u.marcsec, '<', 

414 metadata_query={}) 

415 self.spec_set = SpecificationSet([s1, s2, s3, s4, s5]) 

416 

417 def test_metadata_subset(self): 

418 """Subset by metadata only.""" 

419 subset = self.spec_set.subset(meta={'filter_name': 'r'}) 

420 

421 # In because filter_name: r does not conflict with spec 

422 self.assertIn('validate_drp.AM1.design_r', subset) 

423 # In because spec has no filter_name metadata query 

424 self.assertIn('validate_drp.AM1.design', subset) 

425 # Not in because spec has conflicting filter_name 

426 self.assertNotIn('validate_drp.AM1.design_i', subset) 

427 # Not in because spec requires 'camera' term 

428 self.assertNotIn('validate_drp.AM1.design_HSC_r', subset) 

429 # In because compatible filter_name 

430 self.assertIn('validate_drp.PA1.design_r', subset) 

431 

432 def test_name_and_metadata_subset(self): 

433 """Subset by name and metadata.""" 

434 subset = self.spec_set.subset(name='validate_drp.AM1', 

435 meta={'filter_name': 'r'}) 

436 

437 # Spec has right metric, compatible metadata query 

438 self.assertIn('validate_drp.AM1.design_r', subset) 

439 # Spec has right metric, no metadata query 

440 self.assertIn('validate_drp.AM1.design', subset) 

441 # Conflicting filter_name 

442 self.assertNotIn('validate_drp.AM1.design_i', subset) 

443 # Requires camera term 

444 self.assertNotIn('validate_drp.AM1.design_HSC_r', subset) 

445 # Wrong metric 

446 self.assertNotIn('validate_drp.PA1.design_r', subset) 

447 

448 def test_required_metadata_subset(self): 

449 """Subset by required_meta only.""" 

450 subset = self.spec_set.subset(required_meta={'filter_name': 'r'}) 

451 

452 # In because filter_name: r does not conflict with spec 

453 self.assertIn('validate_drp.AM1.design_r', subset) 

454 # Not in because spec has no filter_name metadata query 

455 self.assertNotIn('validate_drp.AM1.design', subset) 

456 # Not in because spec has conflicting filter_name 

457 self.assertNotIn('validate_drp.AM1.design_i', subset) 

458 # In because spec has compatible filter_name 

459 self.assertIn('validate_drp.AM1.design_HSC_r', subset) 

460 # In because compatible filter_name 

461 self.assertIn('validate_drp.PA1.design_r', subset) 

462 

463 def test_name_and_required_metadata_subset(self): 

464 """Subset by name and required_meta.""" 

465 subset = self.spec_set.subset(name='validate_drp.AM1', 

466 required_meta={'filter_name': 'r'}) 

467 

468 # In because AM1.design_r has `filter_name: r` 

469 self.assertIn('validate_drp.AM1.design_r', subset) 

470 # This AM1.design spec doesn't have a `filter_name` term. 

471 self.assertNotIn('validate_drp.AM1.design', subset) 

472 # Wrong filter_name value 

473 self.assertNotIn('validate_drp.AM1.design_i', subset) 

474 # Has the filer_name: r term; ignores camera term. 

475 self.assertIn('validate_drp.AM1.design_HSC_r', subset) 

476 # Not in because PA1 metric 

477 self.assertNotIn('validate_drp.PA1.design_r', subset) 

478 

479 def test_name_subset(self): 

480 """Subset by name.""" 

481 subset = self.spec_set.subset(name='validate_drp.AM1') 

482 

483 # In because AM1 metric 

484 self.assertIn('validate_drp.AM1.design_r', subset) 

485 self.assertIn('validate_drp.AM1.design_i', subset) 

486 self.assertIn('validate_drp.AM1.design_HSC_r', subset) 

487 # Not in because PA1 metric 

488 self.assertNotIn('validate_drp.PA1.design_HSC_r', subset) 

489 

490 

491if __name__ == "__main__": 491 ↛ 492line 491 didn't jump to line 492, because the condition on line 491 was never true

492 unittest.main()