Coverage for tests/test_indexing.py: 11%

127 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-18 18:35 +0000

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import io 

13import json 

14import os 

15import tempfile 

16import unittest 

17 

18from astro_metadata_translator import ObservationGroup, ObservationInfo 

19from astro_metadata_translator.file_helpers import read_file_info 

20from astro_metadata_translator.indexing import ( 

21 index_files, 

22 process_index_data, 

23 process_sidecar_data, 

24 read_index, 

25 read_sidecar, 

26) 

27 

28TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

29TESTDATA = os.path.join(TESTDIR, "data") 

30 

31 

32class IndexingTestCase(unittest.TestCase): 

33 def test_indexing(self): 

34 """Test that we can index two headers""" 

35 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"] 

36 files = [os.path.join(TESTDATA, f) for f in files] 

37 

38 # Index the translated metadata 

39 index, okay, failed = index_files(files, None, 1, None, "translated") 

40 self.assertEqual(set(files), set(okay)) 

41 self.assertEqual(failed, []) 

42 

43 self.assertIn("__COMMON__", index) 

44 self.assertIn("instrument", index["__COMMON__"]) 

45 

46 # Write the index and check we can read it back. 

47 with tempfile.NamedTemporaryFile(suffix=".json", mode="w+") as temp: 

48 print(json.dumps(index), file=temp) 

49 temp.flush() 

50 externally_processed = read_index(temp.name) 

51 

52 # Convert to an ObservationGroup. Filenames are now stored in the 

53 # corresponding ObservationInfo. 

54 obs_group = process_index_data(index) 

55 self.assertIsInstance(obs_group, ObservationGroup) 

56 self.assertEqual(len(obs_group), 2) 

57 self.assertEqual(obs_group[0].instrument, "HSC") 

58 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files)) 

59 self.assertEqual(externally_processed, obs_group) 

60 

61 metadata = process_index_data(index, force_metadata=True) 

62 self.assertEqual(len(metadata), 2) 

63 self.assertEqual(metadata[files[0]]["instrument"], "HSC") 

64 

65 # Index the native FITS headers 

66 index, okay, failed = index_files(files, None, 1, None, "metadata") 

67 self.assertEqual(set(files), set(okay)) 

68 self.assertEqual(failed, []) 

69 

70 # Check that common entries have been factored out 

71 self.assertIn("__COMMON__", index) 

72 self.assertIn("TELESCOP", index["__COMMON__"]) 

73 self.assertIn("INSTRUME", index[files[0]]) 

74 self.assertNotIn("INSTRUME", index[files[1]]) 

75 self.assertNotIn("TELESCOP", index[files[0]]) 

76 

77 # Convert back to a dict indexed by filename and check that 

78 # common has been put back properly. 

79 metadata = process_index_data(index) 

80 self.assertEqual(len(metadata), 2) 

81 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam") 

82 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

83 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

84 

85 def test_file_reading(self): 

86 """Test the low-level file reader""" 

87 

88 # First with a real header (but YAML) 

89 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml") 

90 info = read_file_info(file, 1, None, "metadata", content_type="simple") 

91 self.assertEqual(info["PROP-ID"], "o15426") 

92 

93 # With metadata sidecar. 

94 json_file = os.path.splitext(file)[0] + ".json" 

95 json_info = read_sidecar(json_file) 

96 # Need to remove the COMMENT fields to avoid confusion between 

97 # PropertyList and the fallback code with multiple entries. 

98 json_info.pop("COMMENT", None) 

99 dict_info = dict(info) # it may be a PropertyList 

100 dict_info.pop("COMMENT", None) 

101 self.assertEqual(json_info, dict_info) 

102 

103 info = read_file_info(file, 1, None, "translated", content_type="native") 

104 self.assertIsInstance(info, ObservationInfo) 

105 self.assertEqual(info.instrument, "HSC") 

106 

107 info = read_file_info(file, 1, None, "translated", content_type="simple") 

108 self.assertIsInstance(info, dict) 

109 self.assertEqual(info["instrument"], "HSC") 

110 

111 json_str = read_file_info(file, 1, None, "translated", content_type="json") 

112 self.assertIsInstance(json_str, str) 

113 info = json.loads(json_str) 

114 self.assertEqual(info["instrument"], "HSC") 

115 

116 processed = process_sidecar_data(info) 

117 self.assertIsInstance(processed, ObservationInfo) 

118 self.assertEqual(processed.instrument, "HSC") 

119 

120 processed = process_sidecar_data(info, force_metadata=True) 

121 self.assertIsInstance(processed, dict) 

122 self.assertEqual(processed["instrument"], "HSC") 

123 

124 json_str = read_file_info(file, 1, None, "metadata", content_type="json") 

125 self.assertIsInstance(json_str, str) 

126 info = json.loads(json_str) 

127 self.assertEqual(info["PROP-ID"], "o15426") 

128 

129 processed = process_sidecar_data(info) 

130 self.assertEqual(processed["PROP-ID"], info["PROP-ID"]) 

131 

132 # Read a small fits file 

133 fits_file = os.path.join(TESTDATA, "small.fits") 

134 info = read_file_info(fits_file, 0, None, "metadata", content_type="native") 

135 self.assertEqual(info["FILTER"], "r") 

136 

137 # The fits file won't translate 

138 with self.assertRaises(ValueError): 

139 read_file_info(fits_file, 0, None, "obsInfo") 

140 

141 with self.assertRaises(ValueError): 

142 read_file_info(file, 1, None, "unknown") 

143 

144 with self.assertRaises(FileNotFoundError): 

145 read_file_info("notthere.not", 1) 

146 

147 with io.StringIO() as err: 

148 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err) 

149 err.seek(0) 

150 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n") 

151 

152 # Now read a file that can not be translated and should trigger 

153 # different errors 

154 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml") 

155 

156 with self.assertLogs(level="DEBUG") as cm: 

157 with self.assertRaises(ValueError): 

158 read_file_info(bad_file, 1) 

159 self.assertIn("Unable to determine translator class", "\n".join(cm.output)) 

160 

161 with io.StringIO() as out: 

162 with io.StringIO() as err: 

163 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out) 

164 out.seek(0) 

165 lines = out.readlines() 

166 self.assertEqual(len(lines), 1) 

167 self.assertIn("ValueError", lines[0]) 

168 

169 with io.StringIO() as out: 

170 with io.StringIO() as err: 

171 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out) 

172 out.seek(0) 

173 lines = out.readlines() 

174 self.assertGreater(len(lines), 4) 

175 self.assertIn("ValueError", lines[-1]) 

176 

177 # A sidecar file that is not a dict. 

178 not_dict = os.path.join(TESTDATA, "bad-sidecar.json") 

179 with self.assertRaises(ValueError): 

180 read_sidecar(not_dict) 

181 

182 with self.assertRaises(ValueError): 

183 read_index(not_dict) 

184 

185 # index file that is not JSON. 

186 with self.assertRaises(ValueError): 

187 read_index(bad_file) 

188 

189 def test_obs_info_sidecar(self): 

190 """Test reading of older files with missing content.""" 

191 

192 # First with a real header (but YAML) 

193 file = os.path.join(TESTDATA, "fitsheader-hsc.yaml") 

194 info = read_file_info(file, 1, None, "translated", content_type="native") 

195 self.assertIsInstance(info, ObservationInfo) 

196 self.assertEqual(info.instrument, "HSC") 

197 

198 # With translated metadata sidecar that lacks the group_counter_start. 

199 json_file = os.path.splitext(file)[0] + ".json" 

200 json_info = read_sidecar(json_file) 

201 self.assertIsInstance(json_info, ObservationInfo) 

202 self.assertEqual(json_info, info) 

203 

204 

205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true

206 unittest.main()