Coverage for tests/test_headers.py: 28%
175 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-14 02:30 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-14 02:30 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import os.path
14import unittest
16from astro_metadata_translator import DecamTranslator, HscTranslator, fix_header, merge_headers
17from astro_metadata_translator.tests import read_test_file
19TESTDIR = os.path.abspath(os.path.dirname(__file__))
22class NotDecamTranslator(DecamTranslator):
23 """This is a DECam translator with override list of header corrections."""
25 name = None
27 @classmethod
28 def fix_header(cls, header, instrument, obsid, filename=None):
29 header["DTSITE"] = "hi"
30 return True
32 @classmethod
33 def translator_version(cls):
34 # Hardcode a version so we can test for it
35 return "1.0.0"
38class NotDecamTranslator2(NotDecamTranslator):
39 """This is like NotDecamTranslator but has a fixup that will break on
40 repeat."""
42 name = None
44 @classmethod
45 def fix_header(cls, header, instrument, obsid, filename=None):
46 header["DTSITE"] += "hi"
47 return True
50class AlsoNotDecamTranslator(DecamTranslator):
51 """This is a DECam translator with override list of header corrections
52 that fails."""
54 name = None
56 @classmethod
57 def fix_header(cls, header, instrument, obsid, filename=None):
58 raise RuntimeError("Failure to work something out from header")
61class NullDecamTranslator(DecamTranslator):
62 """This is a DECam translator that doesn't do any fixes."""
64 name = None
66 @classmethod
67 def fix_header(cls, header, instrument, obsid, filename=None):
68 return False
71class HeadersTestCase(unittest.TestCase):
72 def setUp(self):
73 # Define reference headers
74 self.h1 = dict(
75 ORIGIN="LSST",
76 KEY0=0,
77 KEY1=1,
78 KEY2=3,
79 KEY3=3.1415,
80 KEY4="a",
81 )
83 self.h2 = dict(ORIGIN="LSST", KEY0="0", KEY2=4, KEY5=42)
84 self.h3 = dict(
85 ORIGIN="AUXTEL",
86 KEY3=3.1415,
87 KEY2=50,
88 KEY5=42,
89 )
90 self.h4 = dict(
91 KEY6="New",
92 KEY1="Exists",
93 )
95 # Add keys for sorting by time
96 # Sorted order: h2, h1, h4, h3
97 self.h1["MJD-OBS"] = 50000.0
98 self.h2["MJD-OBS"] = 49000.0
99 self.h3["MJD-OBS"] = 53000.0
100 self.h4["MJD-OBS"] = 52000.0
102 def test_fail(self):
103 with self.assertRaises(ValueError):
104 merge_headers([self.h1, self.h2], mode="wrong")
106 with self.assertRaises(ValueError):
107 merge_headers([])
109 def test_one(self):
110 merged = merge_headers([self.h1], mode="drop")
111 self.assertEqual(merged, self.h1)
113 def test_merging_overwrite(self):
114 merged = merge_headers([self.h1, self.h2], mode="overwrite")
115 # The merged header should be the same type as the first header
116 self.assertIsInstance(merged, type(self.h1))
118 expected = {
119 "MJD-OBS": self.h2["MJD-OBS"],
120 "ORIGIN": self.h2["ORIGIN"],
121 "KEY0": self.h2["KEY0"],
122 "KEY1": self.h1["KEY1"],
123 "KEY2": self.h2["KEY2"],
124 "KEY3": self.h1["KEY3"],
125 "KEY4": self.h1["KEY4"],
126 "KEY5": self.h2["KEY5"],
127 }
128 self.assertEqual(merged, expected)
130 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite")
132 expected = {
133 "MJD-OBS": self.h4["MJD-OBS"],
134 "ORIGIN": self.h3["ORIGIN"],
135 "KEY0": self.h2["KEY0"],
136 "KEY1": self.h4["KEY1"],
137 "KEY2": self.h3["KEY2"],
138 "KEY3": self.h3["KEY3"],
139 "KEY4": self.h1["KEY4"],
140 "KEY5": self.h3["KEY5"],
141 "KEY6": self.h4["KEY6"],
142 }
144 self.assertEqual(merged, expected)
146 def test_merging_first(self):
147 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first")
149 expected = {
150 "MJD-OBS": self.h1["MJD-OBS"],
151 "ORIGIN": self.h1["ORIGIN"],
152 "KEY0": self.h1["KEY0"],
153 "KEY1": self.h1["KEY1"],
154 "KEY2": self.h1["KEY2"],
155 "KEY3": self.h1["KEY3"],
156 "KEY4": self.h1["KEY4"],
157 "KEY5": self.h2["KEY5"],
158 "KEY6": self.h4["KEY6"],
159 }
161 self.assertEqual(merged, expected)
163 def test_merging_drop(self):
164 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop")
166 expected = {
167 "KEY3": self.h1["KEY3"],
168 "KEY4": self.h1["KEY4"],
169 "KEY5": self.h2["KEY5"],
170 "KEY6": self.h4["KEY6"],
171 }
173 self.assertEqual(merged, expected)
175 # Sorting the headers should make no difference to drop mode
176 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop", sort=True)
177 self.assertEqual(merged, expected)
179 # Now retain some headers
180 merged = merge_headers(
181 [self.h1, self.h2, self.h3, self.h4],
182 mode="drop",
183 sort=False,
184 first=["ORIGIN"],
185 last=["KEY2", "KEY1"],
186 )
188 expected = {
189 "KEY2": self.h3["KEY2"],
190 "ORIGIN": self.h1["ORIGIN"],
191 "KEY1": self.h4["KEY1"],
192 "KEY3": self.h1["KEY3"],
193 "KEY4": self.h1["KEY4"],
194 "KEY5": self.h2["KEY5"],
195 "KEY6": self.h4["KEY6"],
196 }
197 self.assertEqual(merged, expected)
199 # Now retain some headers with sorting
200 merged = merge_headers(
201 [self.h1, self.h2, self.h3, self.h4],
202 mode="drop",
203 sort=True,
204 first=["ORIGIN"],
205 last=["KEY2", "KEY1"],
206 )
208 expected = {
209 "KEY2": self.h3["KEY2"],
210 "ORIGIN": self.h2["ORIGIN"],
211 "KEY1": self.h4["KEY1"],
212 "KEY3": self.h1["KEY3"],
213 "KEY4": self.h1["KEY4"],
214 "KEY5": self.h2["KEY5"],
215 "KEY6": self.h4["KEY6"],
216 }
217 self.assertEqual(merged, expected)
219 def test_merging_diff(self):
220 self.maxDiff = None
222 # Nothing in common for diff
223 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="diff")
225 expected = {"__DIFF__": [self.h1, self.h2, self.h3, self.h4]}
227 self.assertEqual(merged, expected)
229 # Now with a subset that does have overlap
230 merged = merge_headers([self.h1, self.h2], mode="diff")
231 expected = {
232 "ORIGIN": "LSST",
233 "__DIFF__": [
234 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
235 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
236 ],
237 }
238 self.assertEqual(merged, expected)
240 # Reverse to make sure there is nothing special about the first header
241 merged = merge_headers([self.h2, self.h1], mode="diff")
242 expected = {
243 "ORIGIN": "LSST",
244 "__DIFF__": [
245 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
246 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
247 ],
248 }
249 self.assertEqual(merged, expected)
251 # Check that identical headers have empty diff
252 merged = merge_headers([self.h1, self.h1], mode="diff")
253 expected = {
254 **self.h1,
255 "__DIFF__": [
256 {},
257 {},
258 ],
259 }
260 self.assertEqual(merged, expected)
262 def test_merging_append(self):
263 # Try with two headers first
264 merged = merge_headers([self.h1, self.h2], mode="append")
266 expected = {
267 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]],
268 "ORIGIN": self.h1["ORIGIN"],
269 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]],
270 "KEY1": self.h1["KEY1"],
271 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]],
272 "KEY3": self.h1["KEY3"],
273 "KEY4": self.h1["KEY4"],
274 "KEY5": self.h2["KEY5"],
275 }
277 self.assertEqual(merged, expected)
279 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append")
281 expected = {
282 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]],
283 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None],
284 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None],
285 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]],
286 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None],
287 "KEY3": self.h3["KEY3"],
288 "KEY4": self.h1["KEY4"],
289 "KEY5": self.h3["KEY5"],
290 "KEY6": self.h4["KEY6"],
291 }
293 self.assertEqual(merged, expected)
295 def test_merging_overwrite_sort(self):
296 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True)
298 expected = {
299 "MJD-OBS": self.h1["MJD-OBS"],
300 "ORIGIN": self.h1["ORIGIN"],
301 "KEY0": self.h1["KEY0"],
302 "KEY1": self.h1["KEY1"],
303 "KEY2": self.h1["KEY2"],
304 "KEY3": self.h1["KEY3"],
305 "KEY4": self.h1["KEY4"],
306 "KEY5": self.h2["KEY5"],
307 }
308 self.assertEqual(merged, expected)
310 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite", sort=True)
312 expected = {
313 "MJD-OBS": self.h3["MJD-OBS"],
314 "ORIGIN": self.h3["ORIGIN"],
315 "KEY0": self.h1["KEY0"],
316 "KEY1": self.h4["KEY1"],
317 "KEY2": self.h3["KEY2"],
318 "KEY3": self.h3["KEY3"],
319 "KEY4": self.h1["KEY4"],
320 "KEY5": self.h3["KEY5"],
321 "KEY6": self.h4["KEY6"],
322 }
324 self.assertEqual(merged, expected)
326 # Changing the order should not change the result
327 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], mode="overwrite", sort=True)
329 self.assertEqual(merged, expected)
331 def test_merging_first_sort(self):
332 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first", sort=True)
334 expected = {
335 "MJD-OBS": self.h2["MJD-OBS"],
336 "ORIGIN": self.h2["ORIGIN"],
337 "KEY0": self.h2["KEY0"],
338 "KEY1": self.h1["KEY1"],
339 "KEY2": self.h2["KEY2"],
340 "KEY3": self.h1["KEY3"],
341 "KEY4": self.h1["KEY4"],
342 "KEY5": self.h2["KEY5"],
343 "KEY6": self.h4["KEY6"],
344 }
346 self.assertEqual(merged, expected)
348 def test_merging_append_sort(self):
349 # Try with two headers first
350 merged = merge_headers([self.h1, self.h2], mode="append", sort=True)
352 expected = {
353 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]],
354 "ORIGIN": self.h1["ORIGIN"],
355 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]],
356 "KEY1": self.h1["KEY1"],
357 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]],
358 "KEY3": self.h1["KEY3"],
359 "KEY4": self.h1["KEY4"],
360 "KEY5": self.h2["KEY5"],
361 }
363 self.assertEqual(merged, expected)
365 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append", sort=True)
367 expected = {
368 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]],
369 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]],
370 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None],
371 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None],
372 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]],
373 "KEY3": self.h3["KEY3"],
374 "KEY4": self.h1["KEY4"],
375 "KEY5": self.h3["KEY5"],
376 "KEY6": self.h4["KEY6"],
377 }
379 self.assertEqual(merged, expected)
381 # Order should not matter
382 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], mode="append", sort=True)
383 self.assertEqual(merged, expected)
386class FixHeadersTestCase(unittest.TestCase):
387 def test_basic_fix_header(self):
388 """Test that a header can be fixed if we specify a local path."""
390 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
391 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3")
393 # First fix header but using no search path (should work as no-op)
394 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
395 self.assertFalse(fixed)
397 # Now using the test corrections directory
398 header2 = copy.copy(header)
399 fixed = fix_header(
400 header2,
401 search_path=os.path.join(TESTDIR, "data", "corrections"),
402 translator_class=NullDecamTranslator,
403 )
404 self.assertTrue(fixed)
405 self.assertEqual(header2["DETECTOR"], "NEW-ID")
407 # Now with a corrections directory that has bad YAML in it
408 header2 = copy.copy(header)
409 with self.assertLogs(level="WARN"):
410 fixed = fix_header(
411 header2,
412 search_path=os.path.join(TESTDIR, "data", "bad_corrections"),
413 translator_class=NullDecamTranslator,
414 )
415 self.assertFalse(fixed)
417 # Test that fix_header of unknown header is allowed
418 header = {"SOMETHING": "UNKNOWN"}
419 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
420 self.assertFalse(fixed)
422 def test_hsc_fix_header(self):
423 """Check that one of the known HSC corrections is being applied
424 properly."""
425 header = {"EXP-ID": "HSCA00120800", "INSTRUME": "HSC", "DATA-TYP": "FLAT"}
427 fixed = fix_header(header, translator_class=HscTranslator)
428 self.assertTrue(fixed)
429 self.assertEqual(header["DATA-TYP"], "OBJECT")
431 # Check provenance
432 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"])
434 # And that this header won't be corrected
435 header = {"EXP-ID": "HSCA00120800X", "INSTRUME": "HSC", "DATA-TYP": "FLAT"}
437 fixed = fix_header(header, translator_class=HscTranslator)
438 self.assertFalse(fixed)
439 self.assertEqual(header["DATA-TYP"], "FLAT")
441 def test_decam_fix_header(self):
442 """Check that one of the known DECam corrections is being applied
443 properly."""
445 # This header is a bias (zero) with an erroneous Y filter
446 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
447 fixed = fix_header(header, translator_class=DecamTranslator)
448 self.assertTrue(fixed)
449 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0")
451 def test_translator_fix_header(self):
452 """Check that translator classes can fix headers."""
454 # Read in a known header
455 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
456 self.assertEqual(header["DTSITE"], "ct")
458 header2 = copy.copy(header)
459 fixed = fix_header(header2, translator_class=NotDecamTranslator)
460 self.assertTrue(fixed)
461 self.assertEqual(header2["DTSITE"], "hi")
463 header2 = copy.copy(header)
464 header2["DTSITE"] = "reset"
465 with self.assertLogs("astro_metadata_translator", level="FATAL"):
466 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator)
467 self.assertFalse(fixed)
468 self.assertEqual(header2["DTSITE"], "reset")
470 def test_no_double_fix(self):
471 """Check that header fixup only happens once."""
473 # Read in a known header
474 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
475 self.assertEqual(header["DTSITE"], "ct")
477 # First time it will modifiy DTSITE
478 fixed = fix_header(header, translator_class=NotDecamTranslator2)
479 self.assertTrue(fixed)
480 self.assertEqual(header["DTSITE"], "cthi")
482 # Get the fix up date
483 date = header["HIERARCH ASTRO METADATA FIX DATE"]
485 # Second time it will do nothing but still report it was fixed
486 fixed = fix_header(header, translator_class=NotDecamTranslator2)
487 self.assertTrue(fixed)
488 self.assertEqual(header["DTSITE"], "cthi")
490 # Date of fixup should be the same
491 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date)
493 # Test the translator version in provenance
494 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0")
497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 unittest.main()