Coverage for tests/test_indexing.py: 16%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import io
13import json
14import os
15import unittest
17from astro_metadata_translator import ObservationInfo, ObservationGroup
18from astro_metadata_translator.indexing import index_files, process_index_data, process_sidecar_data
19from astro_metadata_translator.file_helpers import read_file_info
21TESTDIR = os.path.abspath(os.path.dirname(__file__))
22TESTDATA = os.path.join(TESTDIR, "data")
25class IndexingTestCase(unittest.TestCase):
27 def test_indexing(self):
28 """Test that we can index two headers"""
29 files = ["fitsheader-hsc-HSCA04090107.yaml", "fitsheader-hsc.yaml"]
30 files = [os.path.join(TESTDATA, f) for f in files]
32 # Index the translated metadata
33 index, okay, failed = index_files(files, None, 1, None, "translated")
34 self.assertEqual(set(files), set(okay))
35 self.assertEqual(failed, [])
37 self.assertIn("__COMMON__", index)
38 self.assertIn("instrument", index["__COMMON__"])
40 # Convert to an ObservationGroup. Filenames are now stored in the
41 # corresponding ObservationInfo.
42 obs_group = process_index_data(index)
43 self.assertIsInstance(obs_group, ObservationGroup)
44 self.assertEqual(len(obs_group), 2)
45 self.assertEqual(obs_group[0].instrument, "HSC")
46 self.assertEqual(set([obs_group[0].filename, obs_group[1].filename]), set(files))
48 metadata = process_index_data(index, force_metadata=True)
49 self.assertEqual(len(metadata), 2)
50 self.assertEqual(metadata[files[0]]["instrument"], "HSC")
52 # Index the native FITS headers
53 index, okay, failed = index_files(files, None, 1, None, "metadata")
54 self.assertEqual(set(files), set(okay))
55 self.assertEqual(failed, [])
57 # Check that common entries have been factored out
58 self.assertIn("__COMMON__", index)
59 self.assertIn("TELESCOP", index["__COMMON__"])
60 self.assertIn("INSTRUME", index[files[0]])
61 self.assertNotIn("INSTRUME", index[files[1]])
62 self.assertNotIn("TELESCOP", index[files[0]])
64 # Convert back to a dict indexed by filename and check that
65 # common has been put back properly.
66 metadata = process_index_data(index)
67 self.assertEqual(len(metadata), 2)
68 self.assertEqual(metadata[files[0]]["INSTRUME"], "Hyper Suprime-Cam")
69 self.assertEqual(metadata[files[0]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
70 self.assertEqual(metadata[files[1]]["TELESCOP"], index["__COMMON__"]["TELESCOP"])
72 def test_file_reading(self):
73 """Test the low-level file reader"""
75 # First with a real header (but YAML)
76 file = os.path.join(TESTDATA, "fitsheader-hsc-HSCA04090107.yaml")
77 info = read_file_info(file, 1, None, "metadata", content_type="simple")
78 self.assertEqual(info["PROP-ID"], "o15426")
80 info = read_file_info(file, 1, None, "translated", content_type="native")
81 self.assertIsInstance(info, ObservationInfo)
82 self.assertEqual(info.instrument, "HSC")
84 info = read_file_info(file, 1, None, "translated", content_type="simple")
85 self.assertIsInstance(info, dict)
86 self.assertEqual(info["instrument"], "HSC")
88 json_str = read_file_info(file, 1, None, "translated", content_type="json")
89 self.assertIsInstance(json_str, str)
90 info = json.loads(json_str)
91 self.assertEqual(info["instrument"], "HSC")
93 processed = process_sidecar_data(info)
94 self.assertIsInstance(processed, ObservationInfo)
95 self.assertEqual(processed.instrument, "HSC")
97 processed = process_sidecar_data(info, force_metadata=True)
98 self.assertIsInstance(processed, dict)
99 self.assertEqual(processed["instrument"], "HSC")
101 json_str = read_file_info(file, 1, None, "metadata", content_type="json")
102 self.assertIsInstance(json_str, str)
103 info = json.loads(json_str)
104 self.assertEqual(info["PROP-ID"], "o15426")
106 processed = process_sidecar_data(info)
107 self.assertEqual(processed["PROP-ID"], info["PROP-ID"])
109 # Read a small fits file
110 fits_file = os.path.join(TESTDATA, "small.fits")
111 info = read_file_info(fits_file, 0, None, "metadata", content_type="native")
112 self.assertEqual(info["FILTER"], "r")
114 # The fits file won't translate
115 with self.assertRaises(ValueError):
116 read_file_info(fits_file, 0, None, "obsInfo")
118 with self.assertRaises(ValueError):
119 read_file_info(file, 1, None, "unknown")
121 with self.assertRaises(FileNotFoundError):
122 read_file_info("notthere.not", 1)
124 with io.StringIO() as err:
125 info = read_file_info("notthere.not", 1, print_trace=False, errstream=err)
126 err.seek(0)
127 self.assertEqual(err.readlines()[0], "Unable to open file notthere.not\n")
129 # Now read a file that can not be translated and should trigger
130 # different errors
131 bad_file = os.path.join(TESTDATA, "corrections", "SCUBA_test-20000101_00002.yaml")
133 with self.assertRaises(ValueError):
134 read_file_info(bad_file, 1)
136 with io.StringIO() as out:
137 with io.StringIO() as err:
138 info = read_file_info(bad_file, 1, print_trace=False, errstream=err, outstream=out)
139 out.seek(0)
140 lines = out.readlines()
141 self.assertEqual(len(lines), 1)
142 self.assertIn("ValueError", lines[0])
144 with io.StringIO() as out:
145 with io.StringIO() as err:
146 info = read_file_info(bad_file, 1, print_trace=True, errstream=err, outstream=out)
147 out.seek(0)
148 lines = out.readlines()
149 self.assertGreater(len(lines), 4)
150 self.assertIn("ValueError", lines[-1])
153if __name__ == "__main__": 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 unittest.main()