Coverage for tests/test_indexing.py: 16%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

97 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import io 

13import json 

14import os 

15import unittest 

16 

17from astro_metadata_translator import ObservationGroup, ObservationInfo 

18from astro_metadata_translator.file_helpers import read_file_info 

19from astro_metadata_translator.indexing import index_files, process_index_data, process_sidecar_data 

20 

21TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

22TESTDATA = os.path.join(TESTDIR, "data") 

23 

24 

25class IndexingTestCase(unittest.TestCase): 

26 def test_indexing(self): 

27 """Test that we can index two headers""" 

28 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"] 

29 files = [os.path.join(TESTDATA, f) for f in files] 

30 

31 # Index the translated metadata 

32 index, okay, failed = index_files(files, None, 1, None, "translated") 

33 self.assertEqual(set(files), set(okay)) 

34 self.assertEqual(failed, []) 

35 

36 self.assertIn("__COMMON__", index) 

37 self.assertIn("instrument", index["__COMMON__"]) 

38 

39 # Convert to an ObservationGroup. Filenames are now stored in the 

40 # corresponding ObservationInfo. 

41 obs_group = process_index_data(index) 

42 self.assertIsInstance(obs_group, ObservationGroup) 

43 self.assertEqual(len(obs_group), 2) 

44 self.assertEqual(obs_group[0].instrument, "HSC") 

45 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files)) 

46 

47 metadata = process_index_data(index, force_metadata=True) 

48 self.assertEqual(len(metadata), 2) 

49 self.assertEqual(metadata[files[0]]["instrument"], "HSC") 

50 

51 # Index the native FITS headers 

52 index, okay, failed = index_files(files, None, 1, None, "metadata") 

53 self.assertEqual(set(files), set(okay)) 

54 self.assertEqual(failed, []) 

55 

56 # Check that common entries have been factored out 

57 self.assertIn("__COMMON__", index) 

58 self.assertIn("TELESCOP", index["__COMMON__"]) 

59 self.assertIn("INSTRUME", index[files[0]]) 

60 self.assertNotIn("INSTRUME", index[files[1]]) 

61 self.assertNotIn("TELESCOP", index[files[0]]) 

62 

63 # Convert back to a dict indexed by filename and check that 

64 # common has been put back properly. 

65 metadata = process_index_data(index) 

66 self.assertEqual(len(metadata), 2) 

67 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam") 

68 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

69 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"]) 

70 

71 def test_file_reading(self): 

72 """Test the low-level file reader""" 

73 

74 # First with a real header (but YAML) 

75 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml") 

76 info = read_file_info(file, 1, None, "metadata", content_type="simple") 

77 self.assertEqual(info["PROP-ID"], "o15426") 

78 

79 info = read_file_info(file, 1, None, "translated", content_type="native") 

80 self.assertIsInstance(info, ObservationInfo) 

81 self.assertEqual(info.instrument, "HSC") 

82 

83 info = read_file_info(file, 1, None, "translated", content_type="simple") 

84 self.assertIsInstance(info, dict) 

85 self.assertEqual(info["instrument"], "HSC") 

86 

87 json_str = read_file_info(file, 1, None, "translated", content_type="json") 

88 self.assertIsInstance(json_str, str) 

89 info = json.loads(json_str) 

90 self.assertEqual(info["instrument"], "HSC") 

91 

92 processed = process_sidecar_data(info) 

93 self.assertIsInstance(processed, ObservationInfo) 

94 self.assertEqual(processed.instrument, "HSC") 

95 

96 processed = process_sidecar_data(info, force_metadata=True) 

97 self.assertIsInstance(processed, dict) 

98 self.assertEqual(processed["instrument"], "HSC") 

99 

100 json_str = read_file_info(file, 1, None, "metadata", content_type="json") 

101 self.assertIsInstance(json_str, str) 

102 info = json.loads(json_str) 

103 self.assertEqual(info["PROP-ID"], "o15426") 

104 

105 processed = process_sidecar_data(info) 

106 self.assertEqual(processed["PROP-ID"], info["PROP-ID"]) 

107 

108 # Read a small fits file 

109 fits_file = os.path.join(TESTDATA, "small.fits") 

110 info = read_file_info(fits_file, 0, None, "metadata", content_type="native") 

111 self.assertEqual(info["FILTER"], "r") 

112 

113 # The fits file won't translate 

114 with self.assertRaises(ValueError): 

115 read_file_info(fits_file, 0, None, "obsInfo") 

116 

117 with self.assertRaises(ValueError): 

118 read_file_info(file, 1, None, "unknown") 

119 

120 with self.assertRaises(FileNotFoundError): 

121 read_file_info("notthere.not", 1) 

122 

123 with io.StringIO() as err: 

124 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err) 

125 err.seek(0) 

126 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n") 

127 

128 # Now read a file that can not be translated and should trigger 

129 # different errors 

130 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml") 

131 

132 with self.assertRaises(ValueError): 

133 read_file_info(bad_file, 1) 

134 

135 with io.StringIO() as out: 

136 with io.StringIO() as err: 

137 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out) 

138 out.seek(0) 

139 lines = out.readlines() 

140 self.assertEqual(len(lines), 1) 

141 self.assertIn("ValueError", lines[0]) 

142 

143 with io.StringIO() as out: 

144 with io.StringIO() as err: 

145 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out) 

146 out.seek(0) 

147 lines = out.readlines() 

148 self.assertGreater(len(lines), 4) 

149 self.assertIn("ValueError", lines[-1]) 

150 

151 

152if __name__ == "__main__": 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true

153 unittest.main()