Coverage for tests/test_headers.py: 29%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import unittest
14import os.path
16from astro_metadata_translator import merge_headers, fix_header, HscTranslator
17from astro_metadata_translator.tests import read_test_file
18from astro_metadata_translator import DecamTranslator
20TESTDIR = os.path.abspath(os.path.dirname(__file__))
23class NotDecamTranslator(DecamTranslator):
24 """This is a DECam translator with override list of header corrections."""
25 name = None
27 @classmethod
28 def fix_header(cls, header, instrument, obsid, filename=None):
29 header["DTSITE"] = "hi"
30 return True
32 @classmethod
33 def translator_version(cls):
34 # Hardcode a version so we can test for it
35 return "1.0.0"
38class NotDecamTranslator2(NotDecamTranslator):
39 """This is like NotDecamTranslator but has a fixup that will break on
40 repeat."""
41 name = None
43 @classmethod
44 def fix_header(cls, header, instrument, obsid, filename=None):
45 header["DTSITE"] += "hi"
46 return True
49class AlsoNotDecamTranslator(DecamTranslator):
50 """This is a DECam translator with override list of header corrections
51 that fails."""
52 name = None
54 @classmethod
55 def fix_header(cls, header, instrument, obsid, filename=None):
56 raise RuntimeError("Failure to work something out from header")
59class NullDecamTranslator(DecamTranslator):
60 """This is a DECam translator that doesn't do any fixes."""
61 name = None
63 @classmethod
64 def fix_header(cls, header, instrument, obsid, filename=None):
65 return False
68class HeadersTestCase(unittest.TestCase):
70 def setUp(self):
71 # Define reference headers
72 self.h1 = dict(
73 ORIGIN="LSST",
74 KEY0=0,
75 KEY1=1,
76 KEY2=3,
77 KEY3=3.1415,
78 KEY4="a",
79 )
81 self.h2 = dict(
82 ORIGIN="LSST",
83 KEY0="0",
84 KEY2=4,
85 KEY5=42
86 )
87 self.h3 = dict(
88 ORIGIN="AUXTEL",
89 KEY3=3.1415,
90 KEY2=50,
91 KEY5=42,
92 )
93 self.h4 = dict(
94 KEY6="New",
95 KEY1="Exists",
96 )
98 # Add keys for sorting by time
99 # Sorted order: h2, h1, h4, h3
100 self.h1["MJD-OBS"] = 50000.0
101 self.h2["MJD-OBS"] = 49000.0
102 self.h3["MJD-OBS"] = 53000.0
103 self.h4["MJD-OBS"] = 52000.0
105 def test_fail(self):
106 with self.assertRaises(ValueError):
107 merge_headers([self.h1, self.h2], mode="wrong")
109 with self.assertRaises(ValueError):
110 merge_headers([])
112 def test_one(self):
113 merged = merge_headers([self.h1], mode="drop")
114 self.assertEqual(merged, self.h1)
116 def test_merging_overwrite(self):
117 merged = merge_headers([self.h1, self.h2], mode="overwrite")
118 # The merged header should be the same type as the first header
119 self.assertIsInstance(merged, type(self.h1))
121 expected = {
122 "MJD-OBS": self.h2["MJD-OBS"],
123 "ORIGIN": self.h2["ORIGIN"],
124 "KEY0": self.h2["KEY0"],
125 "KEY1": self.h1["KEY1"],
126 "KEY2": self.h2["KEY2"],
127 "KEY3": self.h1["KEY3"],
128 "KEY4": self.h1["KEY4"],
129 "KEY5": self.h2["KEY5"],
130 }
131 self.assertEqual(merged, expected)
133 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
134 mode="overwrite")
136 expected = {
137 "MJD-OBS": self.h4["MJD-OBS"],
138 "ORIGIN": self.h3["ORIGIN"],
139 "KEY0": self.h2["KEY0"],
140 "KEY1": self.h4["KEY1"],
141 "KEY2": self.h3["KEY2"],
142 "KEY3": self.h3["KEY3"],
143 "KEY4": self.h1["KEY4"],
144 "KEY5": self.h3["KEY5"],
145 "KEY6": self.h4["KEY6"],
146 }
148 self.assertEqual(merged, expected)
150 def test_merging_first(self):
151 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
152 mode="first")
154 expected = {
155 "MJD-OBS": self.h1["MJD-OBS"],
156 "ORIGIN": self.h1["ORIGIN"],
157 "KEY0": self.h1["KEY0"],
158 "KEY1": self.h1["KEY1"],
159 "KEY2": self.h1["KEY2"],
160 "KEY3": self.h1["KEY3"],
161 "KEY4": self.h1["KEY4"],
162 "KEY5": self.h2["KEY5"],
163 "KEY6": self.h4["KEY6"],
164 }
166 self.assertEqual(merged, expected)
168 def test_merging_drop(self):
169 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
170 mode="drop")
172 expected = {
173 "KEY3": self.h1["KEY3"],
174 "KEY4": self.h1["KEY4"],
175 "KEY5": self.h2["KEY5"],
176 "KEY6": self.h4["KEY6"],
177 }
179 self.assertEqual(merged, expected)
181 # Sorting the headers should make no difference to drop mode
182 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
183 mode="drop", sort=True)
184 self.assertEqual(merged, expected)
186 # Now retain some headers
187 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
188 mode="drop", sort=False, first=["ORIGIN"], last=["KEY2", "KEY1"])
190 expected = {
191 "KEY2": self.h3["KEY2"],
192 "ORIGIN": self.h1["ORIGIN"],
193 "KEY1": self.h4["KEY1"],
194 "KEY3": self.h1["KEY3"],
195 "KEY4": self.h1["KEY4"],
196 "KEY5": self.h2["KEY5"],
197 "KEY6": self.h4["KEY6"],
198 }
199 self.assertEqual(merged, expected)
201 # Now retain some headers with sorting
202 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
203 mode="drop", sort=True, first=["ORIGIN"], last=["KEY2", "KEY1"])
205 expected = {
206 "KEY2": self.h3["KEY2"],
207 "ORIGIN": self.h2["ORIGIN"],
208 "KEY1": self.h4["KEY1"],
209 "KEY3": self.h1["KEY3"],
210 "KEY4": self.h1["KEY4"],
211 "KEY5": self.h2["KEY5"],
212 "KEY6": self.h4["KEY6"],
213 }
214 self.assertEqual(merged, expected)
216 def test_merging_diff(self):
217 self.maxDiff = None
219 # Nothing in common for diff
220 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
221 mode="diff")
223 expected = {
224 "__DIFF__": [self.h1, self.h2, self.h3, self.h4]
225 }
227 self.assertEqual(merged, expected)
229 # Now with a subset that does have overlap
230 merged = merge_headers([self.h1, self.h2],
231 mode="diff")
232 expected = {
233 "ORIGIN": "LSST",
234 "__DIFF__": [
235 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
236 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
237 ]
238 }
239 self.assertEqual(merged, expected)
241 # Reverse to make sure there is nothing special about the first header
242 merged = merge_headers([self.h2, self.h1],
243 mode="diff")
244 expected = {
245 "ORIGIN": "LSST",
246 "__DIFF__": [
247 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")},
248 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")},
249 ]
250 }
251 self.assertEqual(merged, expected)
253 # Check that identical headers have empty diff
254 merged = merge_headers([self.h1, self.h1],
255 mode="diff")
256 expected = {
257 **self.h1,
258 "__DIFF__": [
259 {},
260 {},
261 ]
262 }
263 self.assertEqual(merged, expected)
265 def test_merging_append(self):
266 # Try with two headers first
267 merged = merge_headers([self.h1, self.h2], mode="append")
269 expected = {
270 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]],
271 "ORIGIN": self.h1["ORIGIN"],
272 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]],
273 "KEY1": self.h1["KEY1"],
274 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]],
275 "KEY3": self.h1["KEY3"],
276 "KEY4": self.h1["KEY4"],
277 "KEY5": self.h2["KEY5"],
278 }
280 self.assertEqual(merged, expected)
282 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
283 mode="append")
285 expected = {
286 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]],
287 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None],
288 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None],
289 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]],
290 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None],
291 "KEY3": self.h3["KEY3"],
292 "KEY4": self.h1["KEY4"],
293 "KEY5": self.h3["KEY5"],
294 "KEY6": self.h4["KEY6"],
295 }
297 self.assertEqual(merged, expected)
299 def test_merging_overwrite_sort(self):
300 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True)
302 expected = {
303 "MJD-OBS": self.h1["MJD-OBS"],
304 "ORIGIN": self.h1["ORIGIN"],
305 "KEY0": self.h1["KEY0"],
306 "KEY1": self.h1["KEY1"],
307 "KEY2": self.h1["KEY2"],
308 "KEY3": self.h1["KEY3"],
309 "KEY4": self.h1["KEY4"],
310 "KEY5": self.h2["KEY5"],
311 }
312 self.assertEqual(merged, expected)
314 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
315 mode="overwrite", sort=True)
317 expected = {
318 "MJD-OBS": self.h3["MJD-OBS"],
319 "ORIGIN": self.h3["ORIGIN"],
320 "KEY0": self.h1["KEY0"],
321 "KEY1": self.h4["KEY1"],
322 "KEY2": self.h3["KEY2"],
323 "KEY3": self.h3["KEY3"],
324 "KEY4": self.h1["KEY4"],
325 "KEY5": self.h3["KEY5"],
326 "KEY6": self.h4["KEY6"],
327 }
329 self.assertEqual(merged, expected)
331 # Changing the order should not change the result
332 merged = merge_headers([self.h4, self.h1, self.h3, self.h2],
333 mode="overwrite", sort=True)
335 self.assertEqual(merged, expected)
337 def test_merging_first_sort(self):
338 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
339 mode="first", sort=True)
341 expected = {
342 "MJD-OBS": self.h2["MJD-OBS"],
343 "ORIGIN": self.h2["ORIGIN"],
344 "KEY0": self.h2["KEY0"],
345 "KEY1": self.h1["KEY1"],
346 "KEY2": self.h2["KEY2"],
347 "KEY3": self.h1["KEY3"],
348 "KEY4": self.h1["KEY4"],
349 "KEY5": self.h2["KEY5"],
350 "KEY6": self.h4["KEY6"],
351 }
353 self.assertEqual(merged, expected)
355 def test_merging_append_sort(self):
356 # Try with two headers first
357 merged = merge_headers([self.h1, self.h2], mode="append", sort=True)
359 expected = {
360 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]],
361 "ORIGIN": self.h1["ORIGIN"],
362 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]],
363 "KEY1": self.h1["KEY1"],
364 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]],
365 "KEY3": self.h1["KEY3"],
366 "KEY4": self.h1["KEY4"],
367 "KEY5": self.h2["KEY5"],
368 }
370 self.assertEqual(merged, expected)
372 merged = merge_headers([self.h1, self.h2, self.h3, self.h4],
373 mode="append", sort=True)
375 expected = {
376 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]],
377 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]],
378 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None],
379 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None],
380 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]],
381 "KEY3": self.h3["KEY3"],
382 "KEY4": self.h1["KEY4"],
383 "KEY5": self.h3["KEY5"],
384 "KEY6": self.h4["KEY6"],
385 }
387 self.assertEqual(merged, expected)
389 # Order should not matter
390 merged = merge_headers([self.h4, self.h3, self.h2, self.h1],
391 mode="append", sort=True)
392 self.assertEqual(merged, expected)
395class FixHeadersTestCase(unittest.TestCase):
397 def test_basic_fix_header(self):
398 """Test that a header can be fixed if we specify a local path.
399 """
401 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
402 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3")
404 # First fix header but using no search path (should work as no-op)
405 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
406 self.assertFalse(fixed)
408 # Now using the test corrections directory
409 header2 = copy.copy(header)
410 fixed = fix_header(header2, search_path=os.path.join(TESTDIR, "data", "corrections"),
411 translator_class=NullDecamTranslator)
412 self.assertTrue(fixed)
413 self.assertEqual(header2["DETECTOR"], "NEW-ID")
415 # Now with a corrections directory that has bad YAML in it
416 header2 = copy.copy(header)
417 with self.assertLogs(level="WARN"):
418 fixed = fix_header(header2, search_path=os.path.join(TESTDIR, "data", "bad_corrections"),
419 translator_class=NullDecamTranslator)
420 self.assertFalse(fixed)
422 # Test that fix_header of unknown header is allowed
423 header = {"SOMETHING": "UNKNOWN"}
424 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator)
425 self.assertFalse(fixed)
427 def test_hsc_fix_header(self):
428 """Check that one of the known HSC corrections is being applied
429 properly."""
430 header = {"EXP-ID": "HSCA00120800",
431 "INSTRUME": "HSC",
432 "DATA-TYP": "FLAT"}
434 fixed = fix_header(header, translator_class=HscTranslator)
435 self.assertTrue(fixed)
436 self.assertEqual(header["DATA-TYP"], "OBJECT")
438 # Check provenance
439 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"])
441 # And that this header won't be corrected
442 header = {"EXP-ID": "HSCA00120800X",
443 "INSTRUME": "HSC",
444 "DATA-TYP": "FLAT"}
446 fixed = fix_header(header, translator_class=HscTranslator)
447 self.assertFalse(fixed)
448 self.assertEqual(header["DATA-TYP"], "FLAT")
450 def test_decam_fix_header(self):
451 """Check that one of the known DECam corrections is being applied
452 properly."""
454 # This header is a bias (zero) with an erroneous Y filter
455 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
456 fixed = fix_header(header, translator_class=DecamTranslator)
457 self.assertTrue(fixed)
458 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0")
460 def test_translator_fix_header(self):
461 """Check that translator classes can fix headers."""
463 # Read in a known header
464 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
465 self.assertEqual(header["DTSITE"], "ct")
467 header2 = copy.copy(header)
468 fixed = fix_header(header2, translator_class=NotDecamTranslator)
469 self.assertTrue(fixed)
470 self.assertEqual(header2["DTSITE"], "hi")
472 header2 = copy.copy(header)
473 header2["DTSITE"] = "reset"
474 with self.assertLogs("astro_metadata_translator", level="FATAL"):
475 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator)
476 self.assertFalse(fixed)
477 self.assertEqual(header2["DTSITE"], "reset")
479 def test_no_double_fix(self):
480 """Check that header fixup only happens once."""
482 # Read in a known header
483 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data"))
484 self.assertEqual(header["DTSITE"], "ct")
486 # First time it will modifiy DTSITE
487 fixed = fix_header(header, translator_class=NotDecamTranslator2)
488 self.assertTrue(fixed)
489 self.assertEqual(header["DTSITE"], "cthi")
491 # Get the fix up date
492 date = header["HIERARCH ASTRO METADATA FIX DATE"]
494 # Second time it will do nothing but still report it was fixed
495 fixed = fix_header(header, translator_class=NotDecamTranslator2)
496 self.assertTrue(fixed)
497 self.assertEqual(header["DTSITE"], "cthi")
499 # Date of fixup should be the same
500 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date)
502 # Test the translator version in provenance
503 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0")
506if __name__ == "__main__": 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true
507 unittest.main()