Coverage for tests/test_indexing.py: 10%
127 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 11:09 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 11:09 +0000
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import io
13import json
14import os
15import tempfile
16import unittest
18from astro_metadata_translator import ObservationGroup, ObservationInfo
19from astro_metadata_translator.file_helpers import read_file_info
20from astro_metadata_translator.indexing import (
21 index_files,
22 process_index_data,
23 process_sidecar_data,
24 read_index,
25 read_sidecar,
26)
28TESTDIR = os.path.abspath(os.path.dirname(__file__))
29TESTDATA = os.path.join(TESTDIR, "data")
32class IndexingTestCase(unittest.TestCase):
33 """Test indexing and sidecar functionality."""
35 def test_indexing(self):
36 """Test that we can index two headers."""
37 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"]
38 files = [os.path.join(TESTDATA, f) for f in files]
40 # Index the translated metadata
41 index, okay, failed = index_files(files, None, 1, None, "translated")
42 self.assertEqual(set(files), set(okay))
43 self.assertEqual(failed, [])
45 self.assertIn("__COMMON__", index)
46 self.assertIn("instrument", index["__COMMON__"])
48 # Write the index and check we can read it back.
49 with tempfile.NamedTemporaryFile(suffix=".json", mode="w+") as temp:
50 print(json.dumps(index), file=temp)
51 temp.flush()
52 externally_processed = read_index(temp.name)
54 # Convert to an ObservationGroup. Filenames are now stored in the
55 # corresponding ObservationInfo.
56 obs_group = process_index_data(index)
57 self.assertIsInstance(obs_group, ObservationGroup)
58 self.assertEqual(len(obs_group), 2)
59 self.assertEqual(obs_group[0].instrument, "HSC")
60 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files))
61 self.assertEqual(externally_processed, obs_group)
63 metadata = process_index_data(index, force_metadata=True)
64 self.assertEqual(len(metadata), 2)
65 self.assertEqual(metadata[files[0]]["instrument"], "HSC")
67 # Index the native FITS headers
68 index, okay, failed = index_files(files, None, 1, None, "metadata")
69 self.assertEqual(set(files), set(okay))
70 self.assertEqual(failed, [])
72 # Check that common entries have been factored out
73 self.assertIn("__COMMON__", index)
74 self.assertIn("TELESCOP", index["__COMMON__"])
75 self.assertIn("INSTRUME", index[files[0]])
76 self.assertNotIn("INSTRUME", index[files[1]])
77 self.assertNotIn("TELESCOP", index[files[0]])
79 # Convert back to a dict indexed by filename and check that
80 # common has been put back properly.
81 metadata = process_index_data(index)
82 self.assertEqual(len(metadata), 2)
83 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam")
84 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
85 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
87 def test_file_reading(self):
88 """Test the low-level file reader."""
89 # First with a real header (but YAML)
90 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml")
91 info = read_file_info(file, 1, None, "metadata", content_type="simple")
92 self.assertEqual(info["PROP-ID"], "o15426")
94 # With metadata sidecar.
95 json_file = os.path.splitext(file)[0] + ".json"
96 json_info = read_sidecar(json_file)
97 # Need to remove the COMMENT fields to avoid confusion between
98 # PropertyList and the fallback code with multiple entries.
99 json_info.pop("COMMENT", None)
100 dict_info = dict(info) # it may be a PropertyList
101 dict_info.pop("COMMENT", None)
102 self.assertEqual(json_info, dict_info)
104 info = read_file_info(file, 1, None, "translated", content_type="native")
105 self.assertIsInstance(info, ObservationInfo)
106 self.assertEqual(info.instrument, "HSC")
108 info = read_file_info(file, 1, None, "translated", content_type="simple")
109 self.assertIsInstance(info, dict)
110 self.assertEqual(info["instrument"], "HSC")
112 json_str = read_file_info(file, 1, None, "translated", content_type="json")
113 self.assertIsInstance(json_str, str)
114 info = json.loads(json_str)
115 self.assertEqual(info["instrument"], "HSC")
117 processed = process_sidecar_data(info)
118 self.assertIsInstance(processed, ObservationInfo)
119 self.assertEqual(processed.instrument, "HSC")
121 processed = process_sidecar_data(info, force_metadata=True)
122 self.assertIsInstance(processed, dict)
123 self.assertEqual(processed["instrument"], "HSC")
125 json_str = read_file_info(file, 1, None, "metadata", content_type="json")
126 self.assertIsInstance(json_str, str)
127 info = json.loads(json_str)
128 self.assertEqual(info["PROP-ID"], "o15426")
130 processed = process_sidecar_data(info)
131 self.assertEqual(processed["PROP-ID"], info["PROP-ID"])
133 # Read a small fits file
134 fits_file = os.path.join(TESTDATA, "small.fits")
135 info = read_file_info(fits_file, 0, None, "metadata", content_type="native")
136 self.assertEqual(info["FILTER"], "r")
138 # The fits file won't translate
139 with self.assertRaises(ValueError):
140 read_file_info(fits_file, 0, None, "obsInfo")
142 with self.assertRaises(ValueError):
143 read_file_info(file, 1, None, "unknown")
145 with self.assertRaises(FileNotFoundError):
146 read_file_info("notthere.not", 1)
148 with io.StringIO() as err:
149 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err)
150 err.seek(0)
151 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n")
153 # Now read a file that can not be translated and should trigger
154 # different errors
155 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml")
157 with self.assertLogs(level="DEBUG") as cm:
158 with self.assertRaises(ValueError):
159 read_file_info(bad_file, 1)
160 self.assertIn("Unable to determine translator class", "\n".join(cm.output))
162 with io.StringIO() as out:
163 with io.StringIO() as err:
164 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out)
165 out.seek(0)
166 lines = out.readlines()
167 self.assertEqual(len(lines), 1)
168 self.assertIn("ValueError", lines[0])
170 with io.StringIO() as out:
171 with io.StringIO() as err:
172 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out)
173 out.seek(0)
174 lines = out.readlines()
175 self.assertGreater(len(lines), 4)
176 self.assertIn("ValueError", lines[-1])
178 # A sidecar file that is not a dict.
179 not_dict = os.path.join(TESTDATA, "bad-sidecar.json")
180 with self.assertRaises(ValueError):
181 read_sidecar(not_dict)
183 with self.assertRaises(ValueError):
184 read_index(not_dict)
186 # index file that is not JSON.
187 with self.assertRaises(ValueError):
188 read_index(bad_file)
190 def test_obs_info_sidecar(self):
191 """Test reading of older files with missing content."""
192 # First with a real header (but YAML)
193 file = os.path.join(TESTDATA, "fitsheader-hsc.yaml")
194 info = read_file_info(file, 1, None, "translated", content_type="native")
195 self.assertIsInstance(info, ObservationInfo)
196 self.assertEqual(info.instrument, "HSC")
198 # With translated metadata sidecar that lacks the group_counter_start.
199 json_file = os.path.splitext(file)[0] + ".json"
200 json_info = read_sidecar(json_file)
201 self.assertIsInstance(json_info, ObservationInfo)
202 self.assertEqual(json_info, info)
205if __name__ == "__main__": 205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true
206 unittest.main()