Coverage for tests/test_indexing.py: 16%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

97 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import io 

13import json 

14import os 

15import unittest 

16 

17from astro_metadata_translator import ObservationInfo, ObservationGroup 

18from astro_metadata_translator.indexing import index_files, process_index_data, process_sidecar_data 

19from astro_metadata_translator.file_helpers import read_file_info 

20 

21TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

22TESTDATA = os.path.join(TESTDIR, "data") 

23 

24 

25class IndexingTestCase(unittest.TestCase): 

26 

27 def test_indexing(self): 

28 """Test that we can index two headers""" 

29 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"] 

30 files = [os.path.join(TESTDATA, f) for f in files] 

31 

32 # Index the translated metadata 

33 index, okay, failed = index_files(files, None, 1, None, "translated") 

34 self.assertEqual(set(files), set(okay)) 

35 self.assertEqual(failed, []) 

36 

37 self.assertIn("__COMMON__", index) 

38 self.assertIn("instrument", index["__COMMON__"]) 

39 

40 # Convert to an ObservationGroup. Filenames are now stored in the 

41 # corresponding ObservationInfo. 

42 obs_group = process_index_data(index) 

43 self.assertIsInstance(obs_group, ObservationGroup) 

44 self.assertEqual(len(obs_group), 2) 

45 self.assertEqual(obs_group[0].instrument, "HSC") 

46 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files)) 

47 

48 metadata = process_index_data(index, force_metadata=True) 

49 self.assertEqual(len(metadata), 2) 

50 self.assertEqual(metadata[files[0]]["instrument"], "HSC") 

51 

52 # Index the native FITS headers 

53 index, okay, failed = index_files(files, None, 1, None, "metadata") 

54 self.assertEqual(set(files), set(okay)) 

55 self.assertEqual(failed, []) 

56 

57 # Check that common entries have been factored out 

58 self.assertIn("__COMMON__", index) 

59 self.assertIn("TELESCOP", index["__COMMON__"]) 

60 self.assertIn("INSTRUME", index[files[0]]) 

61 self.assertNotIn("INSTRUME", index[files[1]]) 

62 self.assertNotIn("TELESCOP", index[files[0]]) 

63 

64 # Convert back to a dict indexed by filename and check that 

65 # common has been put back properly. 

66 metadata = process_index_data(index) 

67 self.assertEqual(len(metadata), 2) 

68 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam") 

69 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

70 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

71 

72 def test_file_reading(self): 

73 """Test the low-level file reader""" 

74 

75 # First with a real header (but YAML) 

76 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml") 

77 info = read_file_info(file, 1, None, "metadata", content_type="simple") 

78 self.assertEqual(info["PROP-ID"], "o15426") 

79 

80 info = read_file_info(file, 1, None, "translated", content_type="native") 

81 self.assertIsInstance(info, ObservationInfo) 

82 self.assertEqual(info.instrument, "HSC") 

83 

84 info = read_file_info(file, 1, None, "translated", content_type="simple") 

85 self.assertIsInstance(info, dict) 

86 self.assertEqual(info["instrument"], "HSC") 

87 

88 json_str = read_file_info(file, 1, None, "translated", content_type="json") 

89 self.assertIsInstance(json_str, str) 

90 info = json.loads(json_str) 

91 self.assertEqual(info["instrument"], "HSC") 

92 

93 processed = process_sidecar_data(info) 

94 self.assertIsInstance(processed, ObservationInfo) 

95 self.assertEqual(processed.instrument, "HSC") 

96 

97 processed = process_sidecar_data(info, force_metadata=True) 

98 self.assertIsInstance(processed, dict) 

99 self.assertEqual(processed["instrument"], "HSC") 

100 

101 json_str = read_file_info(file, 1, None, "metadata", content_type="json") 

102 self.assertIsInstance(json_str, str) 

103 info = json.loads(json_str) 

104 self.assertEqual(info["PROP-ID"], "o15426") 

105 

106 processed = process_sidecar_data(info) 

107 self.assertEqual(processed["PROP-ID"], info["PROP-ID"]) 

108 

109 # Read a small fits file 

110 fits_file = os.path.join(TESTDATA, "small.fits") 

111 info = read_file_info(fits_file, 0, None, "metadata", content_type="native") 

112 self.assertEqual(info["FILTER"], "r") 

113 

114 # The fits file won't translate 

115 with self.assertRaises(ValueError): 

116 read_file_info(fits_file, 0, None, "obsInfo") 

117 

118 with self.assertRaises(ValueError): 

119 read_file_info(file, 1, None, "unknown") 

120 

121 with self.assertRaises(FileNotFoundError): 

122 read_file_info("notthere.not", 1) 

123 

124 with io.StringIO() as err: 

125 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err) 

126 err.seek(0) 

127 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n") 

128 

129 # Now read a file that can not be translated and should trigger 

130 # different errors 

131 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml") 

132 

133 with self.assertRaises(ValueError): 

134 read_file_info(bad_file, 1) 

135 

136 with io.StringIO() as out: 

137 with io.StringIO() as err: 

138 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out) 

139 out.seek(0) 

140 lines = out.readlines() 

141 self.assertEqual(len(lines), 1) 

142 self.assertIn("ValueError", lines[0]) 

143 

144 with io.StringIO() as out: 

145 with io.StringIO() as err: 

146 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out) 

147 out.seek(0) 

148 lines = out.readlines() 

149 self.assertGreater(len(lines), 4) 

150 self.assertIn("ValueError", lines[-1]) 

151 

152 

153if __name__ == "__main__": 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 unittest.main()