Coverage for tests/test_indexing.py: 10%

127 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-08 03:00 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import io 

13import json 

14import os 

15import tempfile 

16import unittest 

17 

18from astro_metadata_translator import ObservationGroup, ObservationInfo 

19from astro_metadata_translator.file_helpers import read_file_info 

20from astro_metadata_translator.indexing import ( 

21 index_files, 

22 process_index_data, 

23 process_sidecar_data, 

24 read_index, 

25 read_sidecar, 

26) 

27 

28TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

29TESTDATA = os.path.join(TESTDIR, "data") 

30 

31 

32class IndexingTestCase(unittest.TestCase): 

33 """Test indexing and sidecar functionality.""" 

34 

35 def test_indexing(self): 

36 """Test that we can index two headers.""" 

37 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"] 

38 files = [os.path.join(TESTDATA, f) for f in files] 

39 

40 # Index the translated metadata 

41 index, okay, failed = index_files(files, None, 1, None, "translated") 

42 self.assertEqual(set(files), set(okay)) 

43 self.assertEqual(failed, []) 

44 

45 self.assertIn("__COMMON__", index) 

46 self.assertIn("instrument", index["__COMMON__"]) 

47 

48 # Write the index and check we can read it back. 

49 with tempfile.NamedTemporaryFile(suffix=".json", mode="w+") as temp: 

50 print(json.dumps(index), file=temp) 

51 temp.flush() 

52 externally_processed = read_index(temp.name) 

53 

54 # Convert to an ObservationGroup. Filenames are now stored in the 

55 # corresponding ObservationInfo. 

56 obs_group = process_index_data(index) 

57 self.assertIsInstance(obs_group, ObservationGroup) 

58 self.assertEqual(len(obs_group), 2) 

59 self.assertEqual(obs_group[0].instrument, "HSC") 

60 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files)) 

61 self.assertEqual(externally_processed, obs_group) 

62 

63 metadata = process_index_data(index, force_metadata=True) 

64 self.assertEqual(len(metadata), 2) 

65 self.assertEqual(metadata[files[0]]["instrument"], "HSC") 

66 

67 # Index the native FITS headers 

68 index, okay, failed = index_files(files, None, 1, None, "metadata") 

69 self.assertEqual(set(files), set(okay)) 

70 self.assertEqual(failed, []) 

71 

72 # Check that common entries have been factored out 

73 self.assertIn("__COMMON__", index) 

74 self.assertIn("TELESCOP", index["__COMMON__"]) 

75 self.assertIn("INSTRUME", index[files[0]]) 

76 self.assertNotIn("INSTRUME", index[files[1]]) 

77 self.assertNotIn("TELESCOP", index[files[0]]) 

78 

79 # Convert back to a dict indexed by filename and check that 

80 # common has been put back properly. 

81 metadata = process_index_data(index) 

82 self.assertEqual(len(metadata), 2) 

83 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam") 

84 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

85 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

86 

87 def test_file_reading(self): 

88 """Test the low-level file reader.""" 

89 # First with a real header (but YAML) 

90 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml") 

91 info = read_file_info(file, 1, None, "metadata", content_type="simple") 

92 self.assertEqual(info["PROP-ID"], "o15426") 

93 

94 # With metadata sidecar. 

95 json_file = os.path.splitext(file)[0] + ".json" 

96 json_info = read_sidecar(json_file) 

97 # Need to remove the COMMENT fields to avoid confusion between 

98 # PropertyList and the fallback code with multiple entries. 

99 json_info.pop("COMMENT", None) 

100 dict_info = dict(info) # it may be a PropertyList 

101 dict_info.pop("COMMENT", None) 

102 self.assertEqual(json_info, dict_info) 

103 

104 info = read_file_info(file, 1, None, "translated", content_type="native") 

105 self.assertIsInstance(info, ObservationInfo) 

106 self.assertEqual(info.instrument, "HSC") 

107 

108 info = read_file_info(file, 1, None, "translated", content_type="simple") 

109 self.assertIsInstance(info, dict) 

110 self.assertEqual(info["instrument"], "HSC") 

111 

112 json_str = read_file_info(file, 1, None, "translated", content_type="json") 

113 self.assertIsInstance(json_str, str) 

114 info = json.loads(json_str) 

115 self.assertEqual(info["instrument"], "HSC") 

116 

117 processed = process_sidecar_data(info) 

118 self.assertIsInstance(processed, ObservationInfo) 

119 self.assertEqual(processed.instrument, "HSC") 

120 

121 processed = process_sidecar_data(info, force_metadata=True) 

122 self.assertIsInstance(processed, dict) 

123 self.assertEqual(processed["instrument"], "HSC") 

124 

125 json_str = read_file_info(file, 1, None, "metadata", content_type="json") 

126 self.assertIsInstance(json_str, str) 

127 info = json.loads(json_str) 

128 self.assertEqual(info["PROP-ID"], "o15426") 

129 

130 processed = process_sidecar_data(info) 

131 self.assertEqual(processed["PROP-ID"], info["PROP-ID"]) 

132 

133 # Read a small fits file 

134 fits_file = os.path.join(TESTDATA, "small.fits") 

135 info = read_file_info(fits_file, 0, None, "metadata", content_type="native") 

136 self.assertEqual(info["FILTER"], "r") 

137 

138 # The fits file won't translate 

139 with self.assertRaises(ValueError): 

140 read_file_info(fits_file, 0, None, "obsInfo") 

141 

142 with self.assertRaises(ValueError): 

143 read_file_info(file, 1, None, "unknown") 

144 

145 with self.assertRaises(FileNotFoundError): 

146 read_file_info("notthere.not", 1) 

147 

148 with io.StringIO() as err: 

149 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err) 

150 err.seek(0) 

151 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n") 

152 

153 # Now read a file that can not be translated and should trigger 

154 # different errors 

155 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml") 

156 

157 with self.assertLogs(level="DEBUG") as cm: 

158 with self.assertRaises(ValueError): 

159 read_file_info(bad_file, 1) 

160 self.assertIn("Unable to determine translator class", "\n".join(cm.output)) 

161 

162 with io.StringIO() as out: 

163 with io.StringIO() as err: 

164 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out) 

165 out.seek(0) 

166 lines = out.readlines() 

167 self.assertEqual(len(lines), 1) 

168 self.assertIn("ValueError", lines[0]) 

169 

170 with io.StringIO() as out: 

171 with io.StringIO() as err: 

172 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out) 

173 out.seek(0) 

174 lines = out.readlines() 

175 self.assertGreater(len(lines), 4) 

176 self.assertIn("ValueError", lines[-1]) 

177 

178 # A sidecar file that is not a dict. 

179 not_dict = os.path.join(TESTDATA, "bad-sidecar.json") 

180 with self.assertRaises(ValueError): 

181 read_sidecar(not_dict) 

182 

183 with self.assertRaises(ValueError): 

184 read_index(not_dict) 

185 

186 # index file that is not JSON. 

187 with self.assertRaises(ValueError): 

188 read_index(bad_file) 

189 

190 def test_obs_info_sidecar(self): 

191 """Test reading of older files with missing content.""" 

192 # First with a real header (but YAML) 

193 file = os.path.join(TESTDATA, "fitsheader-hsc.yaml") 

194 info = read_file_info(file, 1, None, "translated", content_type="native") 

195 self.assertIsInstance(info, ObservationInfo) 

196 self.assertEqual(info.instrument, "HSC") 

197 

198 # With translated metadata sidecar that lacks the group_counter_start. 

199 json_file = os.path.splitext(file)[0] + ".json" 

200 json_info = read_sidecar(json_file) 

201 self.assertIsInstance(json_info, ObservationInfo) 

202 self.assertEqual(json_info, info) 

203 

204 

205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true

206 unittest.main()