Coverage for tests/test_indexing.py: 10%
127 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 02:35 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 02:35 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import io
13import json
14import os
15import tempfile
16import unittest
18from astro_metadata_translator import ObservationGroup, ObservationInfo
19from astro_metadata_translator.file_helpers import read_file_info
20from astro_metadata_translator.indexing import (
21 index_files,
22 process_index_data,
23 process_sidecar_data,
24 read_index,
25 read_sidecar,
26)
28TESTDIR = os.path.abspath(os.path.dirname(__file__))
29TESTDATA = os.path.join(TESTDIR, "data")
32class IndexingTestCase(unittest.TestCase):
33 def test_indexing(self):
34 """Test that we can index two headers"""
35 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"]
36 files = [os.path.join(TESTDATA, f) for f in files]
38 # Index the translated metadata
39 index, okay, failed = index_files(files, None, 1, None, "translated")
40 self.assertEqual(set(files), set(okay))
41 self.assertEqual(failed, [])
43 self.assertIn("__COMMON__", index)
44 self.assertIn("instrument", index["__COMMON__"])
46 # Write the index and check we can read it back.
47 with tempfile.NamedTemporaryFile(suffix=".json", mode="w+") as temp:
48 print(json.dumps(index), file=temp)
49 temp.flush()
50 externally_processed = read_index(temp.name)
52 # Convert to an ObservationGroup. Filenames are now stored in the
53 # corresponding ObservationInfo.
54 obs_group = process_index_data(index)
55 self.assertIsInstance(obs_group, ObservationGroup)
56 self.assertEqual(len(obs_group), 2)
57 self.assertEqual(obs_group[0].instrument, "HSC")
58 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files))
59 self.assertEqual(externally_processed, obs_group)
61 metadata = process_index_data(index, force_metadata=True)
62 self.assertEqual(len(metadata), 2)
63 self.assertEqual(metadata[files[0]]["instrument"], "HSC")
65 # Index the native FITS headers
66 index, okay, failed = index_files(files, None, 1, None, "metadata")
67 self.assertEqual(set(files), set(okay))
68 self.assertEqual(failed, [])
70 # Check that common entries have been factored out
71 self.assertIn("__COMMON__", index)
72 self.assertIn("TELESCOP", index["__COMMON__"])
73 self.assertIn("INSTRUME", index[files[0]])
74 self.assertNotIn("INSTRUME", index[files[1]])
75 self.assertNotIn("TELESCOP", index[files[0]])
77 # Convert back to a dict indexed by filename and check that
78 # common has been put back properly.
79 metadata = process_index_data(index)
80 self.assertEqual(len(metadata), 2)
81 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam")
82 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
83 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
85 def test_file_reading(self):
86 """Test the low-level file reader"""
88 # First with a real header (but YAML)
89 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml")
90 info = read_file_info(file, 1, None, "metadata", content_type="simple")
91 self.assertEqual(info["PROP-ID"], "o15426")
93 # With metadata sidecar.
94 json_file = os.path.splitext(file)[0] + ".json"
95 json_info = read_sidecar(json_file)
96 # Need to remove the COMMENT fields to avoid confusion between
97 # PropertyList and the fallback code with multiple entries.
98 json_info.pop("COMMENT", None)
99 dict_info = dict(info) # it may be a PropertyList
100 dict_info.pop("COMMENT", None)
101 self.assertEqual(json_info, dict_info)
103 info = read_file_info(file, 1, None, "translated", content_type="native")
104 self.assertIsInstance(info, ObservationInfo)
105 self.assertEqual(info.instrument, "HSC")
107 info = read_file_info(file, 1, None, "translated", content_type="simple")
108 self.assertIsInstance(info, dict)
109 self.assertEqual(info["instrument"], "HSC")
111 json_str = read_file_info(file, 1, None, "translated", content_type="json")
112 self.assertIsInstance(json_str, str)
113 info = json.loads(json_str)
114 self.assertEqual(info["instrument"], "HSC")
116 processed = process_sidecar_data(info)
117 self.assertIsInstance(processed, ObservationInfo)
118 self.assertEqual(processed.instrument, "HSC")
120 processed = process_sidecar_data(info, force_metadata=True)
121 self.assertIsInstance(processed, dict)
122 self.assertEqual(processed["instrument"], "HSC")
124 json_str = read_file_info(file, 1, None, "metadata", content_type="json")
125 self.assertIsInstance(json_str, str)
126 info = json.loads(json_str)
127 self.assertEqual(info["PROP-ID"], "o15426")
129 processed = process_sidecar_data(info)
130 self.assertEqual(processed["PROP-ID"], info["PROP-ID"])
132 # Read a small fits file
133 fits_file = os.path.join(TESTDATA, "small.fits")
134 info = read_file_info(fits_file, 0, None, "metadata", content_type="native")
135 self.assertEqual(info["FILTER"], "r")
137 # The fits file won't translate
138 with self.assertRaises(ValueError):
139 read_file_info(fits_file, 0, None, "obsInfo")
141 with self.assertRaises(ValueError):
142 read_file_info(file, 1, None, "unknown")
144 with self.assertRaises(FileNotFoundError):
145 read_file_info("notthere.not", 1)
147 with io.StringIO() as err:
148 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err)
149 err.seek(0)
150 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n")
152 # Now read a file that can not be translated and should trigger
153 # different errors
154 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml")
156 with self.assertLogs(level="DEBUG") as cm:
157 with self.assertRaises(ValueError):
158 read_file_info(bad_file, 1)
159 self.assertIn("Unable to determine translator class", "\n".join(cm.output))
161 with io.StringIO() as out:
162 with io.StringIO() as err:
163 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out)
164 out.seek(0)
165 lines = out.readlines()
166 self.assertEqual(len(lines), 1)
167 self.assertIn("ValueError", lines[0])
169 with io.StringIO() as out:
170 with io.StringIO() as err:
171 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out)
172 out.seek(0)
173 lines = out.readlines()
174 self.assertGreater(len(lines), 4)
175 self.assertIn("ValueError", lines[-1])
177 # A sidecar file that is not a dict.
178 not_dict = os.path.join(TESTDATA, "bad-sidecar.json")
179 with self.assertRaises(ValueError):
180 read_sidecar(not_dict)
182 with self.assertRaises(ValueError):
183 read_index(not_dict)
185 # index file that is not JSON.
186 with self.assertRaises(ValueError):
187 read_index(bad_file)
189 def test_obs_info_sidecar(self):
190 """Test reading of older files with missing content."""
192 # First with a real header (but YAML)
193 file = os.path.join(TESTDATA, "fitsheader-hsc.yaml")
194 info = read_file_info(file, 1, None, "translated", content_type="native")
195 self.assertIsInstance(info, ObservationInfo)
196 self.assertEqual(info.instrument, "HSC")
198 # With translated metadata sidecar that lacks the group_counter_start.
199 json_file = os.path.splitext(file)[0] + ".json"
200 json_info = read_sidecar(json_file)
201 self.assertIsInstance(json_info, ObservationInfo)
202 self.assertEqual(json_info, info)
205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true
206 unittest.main()