Coverage for tests/test_headers.py: 23%
175 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-06 02:41 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-06 02:41 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import os.path
14import unittest
16from astro_metadata_translator import DecamTranslator, HscTranslator, fix_header, merge_headers
17from astro_metadata_translator.tests import read_test_file
19TESTDIR = os.path.abspath(os.path.dirname(__file__))
22class NotDecamTranslator(DecamTranslator):
23 """A DECam translator with override list of header corrections."""
25 name = None
27 @classmethod
28 def fix_header(cls, header, instrument, obsid, filename=None):
29 header["DTSITE"] = "hi"
30 return True
32 @classmethod
33 def translator_version(cls):
34 # Hardcode a version so we can test for it
35 return "1.0.0"
38class NotDecamTranslator2(NotDecamTranslator):
39 """Similar to NotDecamTranslator but has a fixup that will break on
40 repeat.
41 """
43 name = None
45 @classmethod
46 def fix_header(cls, header, instrument, obsid, filename=None):
47 header["DTSITE"] += "hi"
48 return True
51class AlsoNotDecamTranslator(DecamTranslator):
52 """A DECam translator with override list of header corrections
53 that fails.
54 """
56 name = None
58 @classmethod
59 def fix_header(cls, header, instrument, obsid, filename=None):
60 raise RuntimeError("Failure to work something out from header")
63class NullDecamTranslator(DecamTranslator):
64 """A DECam translator that doesn't do any fixes."""
66 name = None
68 @classmethod
69 def fix_header(cls, header, instrument, obsid, filename=None):
70 return False
73class HeadersTestCase(unittest.TestCase):
74 """Test header manipulation utilities."""
76 def setUp(self):
77 # Define reference headers
78 self.h1 = dict(
79 ORIGIN="LSST",
80 KEY0=0,
81 KEY1=1,
82 KEY2=3,
83 KEY3=3.1415,
84 KEY4="a",
85 )
87 self.h2 = dict(ORIGIN="LSST", KEY0="0", KEY2=4, KEY5=42)
88 self.h3 = dict(
89 ORIGIN="AUXTEL",
90 KEY3=3.1415,
91 KEY2=50,
92 KEY5=42,
93 )
94 self.h4 = dict(
95 KEY6="New",
96 KEY1="Exists",
97 )
99 # Add keys for sorting by time
100 # Sorted order: h2, h1, h4, h3
101 self.h1["MJD-OBS"] = 50000.0
102 self.h2["MJD-OBS"] = 49000.0
103 self.h3["MJD-OBS"] = 53000.0
104 self.h4["MJD-OBS"] = 52000.0
106 def test_fail(self):
107 with self.assertRaises(ValueError):
108 merge_headers([self.h1, self.h2], mode="wrong")
110 with self.assertRaises(ValueError):
111 merge_headers([])
113 def test_one(self):
114 merged = merge_headers([self.h1], mode="drop")
115 self.assertEqual(merged, self.h1)
117 def test_merging_overwrite(self):
118 merged = merge_headers([self.h1, self.h2], mode="overwrite")
119 # The merged header should be the same type as the first header
120 self.assertIsInstance(merged, type(self.h1))
122 expected = {
123 "MJD-OBS": self.h2["MJD-OBS"],
124 "ORIGIN": self.h2["ORIGIN"],
125 "KEY0": self.h2["KEY0"],
126 "KEY1": self.h1["KEY1"],
127 "KEY2": self.h2["KEY2"],
128 "KEY3": self.h1["KEY3"],
129 "KEY4": self.h1["KEY4"],
130 "KEY5": self.h2["KEY5"],
131 }
132 self.assertEqual(merged, expected)
134 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite")
136 expected = {
137 "MJD-OBS": self.h4["MJD-OBS"],
138 "ORIGIN": self.h3["ORIGIN"],
139 "KEY0": self.h2["KEY0"],
140 "KEY1": self.h4["KEY1"],
141 "KEY2": self.h3["KEY2"],
142 "KEY3": self.h3["KEY3"],
143 "KEY4": self.h1["KEY4"],
144 "KEY5": self.h3["KEY5"],
145 "KEY6": self.h4["KEY6"],
146 }
148 self.assertEqual(merged, expected)
150 def test_merging_first(self):
151 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first")
153 expected = {
154 "MJD-OBS": self.h1["MJD-OBS"],
155 "ORIGIN": self.h1["ORIGIN"],
156 "KEY0": self.h1["KEY0"],
157 "KEY1": self.h1["KEY1"],
158 "KEY2": self.h1["KEY2"],
159 "KEY3": self.h1["KEY3"],
160 "KEY4": self.h1["KEY4"],
161 "KEY5": self.h2["KEY5"],
162 "KEY6": self.h4["KEY6"],
163 }
165 self.assertEqual(merged, expected)
167 def test_merging_drop(self):
168 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop")
170 expected = {
171 "KEY3": self.h1["KEY3"],
172 "KEY4": self.h1["KEY4"],
173 "KEY5": self.h2["KEY5"],
174 "KEY6": self.h4["KEY6"],
175 }
177 self.assertEqual(merged, expected)
179 # Sorting the headers should make no difference to drop mode
180 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop", sort=True)
181 self.assertEqual(merged, expected)
183 # Now retain some headers
184 merged = merge_headers(
185 [self.h1, self.h2, self.h3, self.h4],
186 mode="drop",
187 sort=False,
188 first=["ORIGIN"],
189 last=["KEY2", "KEY1"],
190 )
192 expected = {
193 "KEY2": self.h3["KEY2"],
194 "ORIGIN": self.h1["ORIGIN"],
195 "KEY1": self.h4["KEY1"],
196 "KEY3": self.h1["KEY3"],
197 "KEY4": self.h1["KEY4"],
198 "KEY5": self.h2["KEY5"],
199 "KEY6": self.h4["KEY6"],
200 }
201 self.assertEqual(merged, expected)
203 # Now retain some headers with sorting
204 merged = merge_headers(
205 [self.h1, self.h2, self.h3, self.h4],
206 mode="drop",
207 sort=True,
208 first=["ORIGIN"],
209 last=["KEY2", "KEY1"],
210 )
212 expected = {
213 "KEY2": self.h3["KEY2"],
214 "ORIGIN": self.h2["ORIGIN"],
215 "KEY1": self.h4["KEY1"],
216 "KEY3": self.h1["KEY3"],
217 "KEY4": self.h1["KEY4"],
218 "KEY5": self.h2["KEY5"],
219 "KEY6": self.h4["KEY6"],
220 }
221 self.assertEqual(merged, expected)
223 def test_merging_diff(self):
224 self.maxDiff = None
226 # Nothing in common for diff
227 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="diff")
229 expected = {"__DIFF__": [self.h1, self.h2, self.h3, self.h4]}
231 self.assertEqual(merged, expected)
233 # Now with a subset that does have overlap
234 merged = merge_headers([self.h1, self.h2], mode="diff")
235 expected = {
236 "ORIGIN": "LSST",
237 "__DIFF__": [
238 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
239 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
240 ],
241 }
242 self.assertEqual(merged, expected)
244 # Reverse to make sure there is nothing special about the first header
245 merged = merge_headers([self.h2, self.h1], mode="diff")
246 expected = {
247 "ORIGIN": "LSST",
248 "__DIFF__": [
249 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
250 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
251 ],
252 }
253 self.assertEqual(merged, expected)
255 # Check that identical headers have empty diff
256 merged = merge_headers([self.h1, self.h1], mode="diff")
257 expected = {
258 **self.h1,
259 "__DIFF__": [
260 {},
261 {},
262 ],
263 }
264 self.assertEqual(merged, expected)
266 def test_merging_append(self):
267 # Try with two headers first
268 merged = merge_headers([self.h1, self.h2], mode="append")
270 expected = {
271 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]],
272 "ORIGIN": self.h1["ORIGIN"],
273 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]],
274 "KEY1": self.h1["KEY1"],
275 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]],
276 "KEY3": self.h1["KEY3"],
277 "KEY4": self.h1["KEY4"],
278 "KEY5": self.h2["KEY5"],
279 }
281 self.assertEqual(merged, expected)
283 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append")
285 expected = {
286 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]],
287 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None],
288 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None],
289 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]],
290 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None],
291 "KEY3": self.h3["KEY3"],
292 "KEY4": self.h1["KEY4"],
293 "KEY5": self.h3["KEY5"],
294 "KEY6": self.h4["KEY6"],
295 }
297 self.assertEqual(merged, expected)
299 def test_merging_overwrite_sort(self):
300 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True)
302 expected = {
303 "MJD-OBS": self.h1["MJD-OBS"],
304 "ORIGIN": self.h1["ORIGIN"],
305 "KEY0": self.h1["KEY0"],
306 "KEY1": self.h1["KEY1"],
307 "KEY2": self.h1["KEY2"],
308 "KEY3": self.h1["KEY3"],
309 "KEY4": self.h1["KEY4"],
310 "KEY5": self.h2["KEY5"],
311 }
312 self.assertEqual(merged, expected)
314 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite", sort=True)
316 expected = {
317 "MJD-OBS": self.h3["MJD-OBS"],
318 "ORIGIN": self.h3["ORIGIN"],
319 "KEY0": self.h1["KEY0"],
320 "KEY1": self.h4["KEY1"],
321 "KEY2": self.h3["KEY2"],
322 "KEY3": self.h3["KEY3"],
323 "KEY4": self.h1["KEY4"],
324 "KEY5": self.h3["KEY5"],
325 "KEY6": self.h4["KEY6"],
326 }
328 self.assertEqual(merged, expected)
330 # Changing the order should not change the result
331 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], mode="overwrite", sort=True)
333 self.assertEqual(merged, expected)
335 def test_merging_first_sort(self):
336 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first", sort=True)
338 expected = {
339 "MJD-OBS": self.h2["MJD-OBS"],
340 "ORIGIN": self.h2["ORIGIN"],
341 "KEY0": self.h2["KEY0"],
342 "KEY1": self.h1["KEY1"],
343 "KEY2": self.h2["KEY2"],
344 "KEY3": self.h1["KEY3"],
345 "KEY4": self.h1["KEY4"],
346 "KEY5": self.h2["KEY5"],
347 "KEY6": self.h4["KEY6"],
348 }
350 self.assertEqual(merged, expected)
352 def test_merging_append_sort(self):
353 # Try with two headers first
354 merged = merge_headers([self.h1, self.h2], mode="append", sort=True)
356 expected = {
357 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]],
358 "ORIGIN": self.h1["ORIGIN"],
359 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]],
360 "KEY1": self.h1["KEY1"],
361 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]],
362 "KEY3": self.h1["KEY3"],
363 "KEY4": self.h1["KEY4"],
364 "KEY5": self.h2["KEY5"],
365 }
367 self.assertEqual(merged, expected)
369 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append", sort=True)
371 expected = {
372 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]],
373 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]],
374 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None],
375 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None],
376 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]],
377 "KEY3": self.h3["KEY3"],
378 "KEY4": self.h1["KEY4"],
379 "KEY5": self.h3["KEY5"],
380 "KEY6": self.h4["KEY6"],
381 }
383 self.assertEqual(merged, expected)
385 # Order should not matter
386 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], mode="append", sort=True)
387 self.assertEqual(merged, expected)
390class FixHeadersTestCase(unittest.TestCase):
391 """Test header fix up."""
393 def test_basic_fix_header(self):
394 """Test that a header can be fixed if we specify a local path."""
395 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
396 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3")
398 # First fix header but using no search path (should work as no-op)
399 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
400 self.assertFalse(fixed)
402 # Now using the test corrections directory
403 header2 = copy.copy(header)
404 fixed = fix_header(
405 header2,
406 search_path=os.path.join(TESTDIR, "data", "corrections"),
407 translator_class=NullDecamTranslator,
408 )
409 self.assertTrue(fixed)
410 self.assertEqual(header2["DETECTOR"], "NEW-ID")
412 # Now with a corrections directory that has bad YAML in it
413 header2 = copy.copy(header)
414 with self.assertLogs(level="WARN"):
415 fixed = fix_header(
416 header2,
417 search_path=os.path.join(TESTDIR, "data", "bad_corrections"),
418 translator_class=NullDecamTranslator,
419 )
420 self.assertFalse(fixed)
422 # Test that fix_header of unknown header is allowed
423 header = {"SOMETHING": "UNKNOWN"}
424 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
425 self.assertFalse(fixed)
427 def test_hsc_fix_header(self):
428 """Check that one of the known HSC corrections is being applied
429 properly.
430 """
431 header = {"EXP-ID": "HSCA00120800", "INSTRUME": "HSC", "DATA-TYP": "FLAT"}
433 fixed = fix_header(header, translator_class=HscTranslator)
434 self.assertTrue(fixed)
435 self.assertEqual(header["DATA-TYP"], "OBJECT")
437 # Check provenance
438 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"])
440 # And that this header won't be corrected
441 header = {"EXP-ID": "HSCA00120800X", "INSTRUME": "HSC", "DATA-TYP": "FLAT"}
443 fixed = fix_header(header, translator_class=HscTranslator)
444 self.assertFalse(fixed)
445 self.assertEqual(header["DATA-TYP"], "FLAT")
447 def test_decam_fix_header(self):
448 """Check that one of the known DECam corrections is being applied
449 properly.
450 """
451 # This header is a bias (zero) with an erroneous Y filter
452 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
453 fixed = fix_header(header, translator_class=DecamTranslator)
454 self.assertTrue(fixed)
455 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0")
457 def test_translator_fix_header(self):
458 """Check that translator classes can fix headers."""
459 # Read in a known header
460 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
461 self.assertEqual(header["DTSITE"], "ct")
463 header2 = copy.copy(header)
464 fixed = fix_header(header2, translator_class=NotDecamTranslator)
465 self.assertTrue(fixed)
466 self.assertEqual(header2["DTSITE"], "hi")
468 header2 = copy.copy(header)
469 header2["DTSITE"] = "reset"
470 with self.assertLogs("astro_metadata_translator", level="FATAL"):
471 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator)
472 self.assertFalse(fixed)
473 self.assertEqual(header2["DTSITE"], "reset")
475 def test_no_double_fix(self):
476 """Check that header fixup only happens once."""
477 # Read in a known header
478 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
479 self.assertEqual(header["DTSITE"], "ct")
481 # First time it will modifiy DTSITE
482 fixed = fix_header(header, translator_class=NotDecamTranslator2)
483 self.assertTrue(fixed)
484 self.assertEqual(header["DTSITE"], "cthi")
486 # Get the fix up date
487 date = header["HIERARCH ASTRO METADATA FIX DATE"]
489 # Second time it will do nothing but still report it was fixed
490 fixed = fix_header(header, translator_class=NotDecamTranslator2)
491 self.assertTrue(fixed)
492 self.assertEqual(header["DTSITE"], "cthi")
494 # Date of fixup should be the same
495 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date)
497 # Test the translator version in provenance
498 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0")
501if __name__ == "__main__": 501 ↛ 502line 501 didn't jump to line 502, because the condition on line 501 was never true
502 unittest.main()