Coverage for tests/test_indexing.py: 16%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import io
13import json
14import os
15import unittest
17from astro_metadata_translator import ObservationGroup, ObservationInfo
18from astro_metadata_translator.file_helpers import read_file_info
19from astro_metadata_translator.indexing import index_files, process_index_data, process_sidecar_data
21TESTDIR = os.path.abspath(os.path.dirname(__file__))
22TESTDATA = os.path.join(TESTDIR, "data")
25class IndexingTestCase(unittest.TestCase):
26 def test_indexing(self):
27 """Test that we can index two headers"""
28 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"]
29 files = [os.path.join(TESTDATA, f) for f in files]
31 # Index the translated metadata
32 index, okay, failed = index_files(files, None, 1, None, "translated")
33 self.assertEqual(set(files), set(okay))
34 self.assertEqual(failed, [])
36 self.assertIn("__COMMON__", index)
37 self.assertIn("instrument", index["__COMMON__"])
39 # Convert to an ObservationGroup. Filenames are now stored in the
40 # corresponding ObservationInfo.
41 obs_group = process_index_data(index)
42 self.assertIsInstance(obs_group, ObservationGroup)
43 self.assertEqual(len(obs_group), 2)
44 self.assertEqual(obs_group[0].instrument, "HSC")
45 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files))
47 metadata = process_index_data(index, force_metadata=True)
48 self.assertEqual(len(metadata), 2)
49 self.assertEqual(metadata[files[0]]["instrument"], "HSC")
51 # Index the native FITS headers
52 index, okay, failed = index_files(files, None, 1, None, "metadata")
53 self.assertEqual(set(files), set(okay))
54 self.assertEqual(failed, [])
56 # Check that common entries have been factored out
57 self.assertIn("__COMMON__", index)
58 self.assertIn("TELESCOP", index["__COMMON__"])
59 self.assertIn("INSTRUME", index[files[0]])
60 self.assertNotIn("INSTRUME", index[files[1]])
61 self.assertNotIn("TELESCOP", index[files[0]])
63 # Convert back to a dict indexed by filename and check that
64 # common has been put back properly.
65 metadata = process_index_data(index)
66 self.assertEqual(len(metadata), 2)
67 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam")
68 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
69 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
71 def test_file_reading(self):
72 """Test the low-level file reader"""
74 # First with a real header (but YAML)
75 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml")
76 info = read_file_info(file, 1, None, "metadata", content_type="simple")
77 self.assertEqual(info["PROP-ID"], "o15426")
79 info = read_file_info(file, 1, None, "translated", content_type="native")
80 self.assertIsInstance(info, ObservationInfo)
81 self.assertEqual(info.instrument, "HSC")
83 info = read_file_info(file, 1, None, "translated", content_type="simple")
84 self.assertIsInstance(info, dict)
85 self.assertEqual(info["instrument"], "HSC")
87 json_str = read_file_info(file, 1, None, "translated", content_type="json")
88 self.assertIsInstance(json_str, str)
89 info = json.loads(json_str)
90 self.assertEqual(info["instrument"], "HSC")
92 processed = process_sidecar_data(info)
93 self.assertIsInstance(processed, ObservationInfo)
94 self.assertEqual(processed.instrument, "HSC")
96 processed = process_sidecar_data(info, force_metadata=True)
97 self.assertIsInstance(processed, dict)
98 self.assertEqual(processed["instrument"], "HSC")
100 json_str = read_file_info(file, 1, None, "metadata", content_type="json")
101 self.assertIsInstance(json_str, str)
102 info = json.loads(json_str)
103 self.assertEqual(info["PROP-ID"], "o15426")
105 processed = process_sidecar_data(info)
106 self.assertEqual(processed["PROP-ID"], info["PROP-ID"])
108 # Read a small fits file
109 fits_file = os.path.join(TESTDATA, "small.fits")
110 info = read_file_info(fits_file, 0, None, "metadata", content_type="native")
111 self.assertEqual(info["FILTER"], "r")
113 # The fits file won't translate
114 with self.assertRaises(ValueError):
115 read_file_info(fits_file, 0, None, "obsInfo")
117 with self.assertRaises(ValueError):
118 read_file_info(file, 1, None, "unknown")
120 with self.assertRaises(FileNotFoundError):
121 read_file_info("notthere.not", 1)
123 with io.StringIO() as err:
124 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err)
125 err.seek(0)
126 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n")
128 # Now read a file that can not be translated and should trigger
129 # different errors
130 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml")
132 with self.assertRaises(ValueError):
133 read_file_info(bad_file, 1)
135 with io.StringIO() as out:
136 with io.StringIO() as err:
137 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out)
138 out.seek(0)
139 lines = out.readlines()
140 self.assertEqual(len(lines), 1)
141 self.assertIn("ValueError", lines[0])
143 with io.StringIO() as out:
144 with io.StringIO() as err:
145 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out)
146 out.seek(0)
147 lines = out.readlines()
148 self.assertGreater(len(lines), 4)
149 self.assertIn("ValueError", lines[-1])
152if __name__ == "__main__": 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true
153 unittest.main()