Coverage for tests/test_headers.py: 25%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

176 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import unittest 

14import os.path 

15 

16from astro_metadata_translator import merge_headers, fix_header, HscTranslator 

17from astro_metadata_translator.tests import read_test_file 

18from astro_metadata_translator import DecamTranslator 

19 

20TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

21 

22 

23class NotDecamTranslator(DecamTranslator): 

24 """This is a DECam translator with override list of header corrections.""" 

25 name = None 

26 

27 @classmethod 

28 def fix_header(cls, header, instrument, obsid, filename=None): 

29 header["DTSITE"] = "hi" 

30 return True 

31 

32 @classmethod 

33 def translator_version(cls): 

34 # Hardcode a version so we can test for it 

35 return "1.0.0" 

36 

37 

38class NotDecamTranslator2(NotDecamTranslator): 

39 """This is like NotDecamTranslator but has a fixup that will break on 

40 repeat.""" 

41 name = None 

42 

43 @classmethod 

44 def fix_header(cls, header, instrument, obsid, filename=None): 

45 header["DTSITE"] += "hi" 

46 return True 

47 

48 

49class AlsoNotDecamTranslator(DecamTranslator): 

50 """This is a DECam translator with override list of header corrections 

51 that fails.""" 

52 name = None 

53 

54 @classmethod 

55 def fix_header(cls, header, instrument, obsid, filename=None): 

56 raise RuntimeError("Failure to work something out from header") 

57 

58 

59class NullDecamTranslator(DecamTranslator): 

60 """This is a DECam translator that doesn't do any fixes.""" 

61 name = None 

62 

63 @classmethod 

64 def fix_header(cls, header, instrument, obsid, filename=None): 

65 return False 

66 

67 

68class HeadersTestCase(unittest.TestCase): 

69 

70 def setUp(self): 

71 # Define reference headers 

72 self.h1 = dict( 

73 ORIGIN="LSST", 

74 KEY0=0, 

75 KEY1=1, 

76 KEY2=3, 

77 KEY3=3.1415, 

78 KEY4="a", 

79 ) 

80 

81 self.h2 = dict( 

82 ORIGIN="LSST", 

83 KEY0="0", 

84 KEY2=4, 

85 KEY5=42 

86 ) 

87 self.h3 = dict( 

88 ORIGIN="AUXTEL", 

89 KEY3=3.1415, 

90 KEY2=50, 

91 KEY5=42, 

92 ) 

93 self.h4 = dict( 

94 KEY6="New", 

95 KEY1="Exists", 

96 ) 

97 

98 # Add keys for sorting by time 

99 # Sorted order: h2, h1, h4, h3 

100 self.h1["MJD-OBS"] = 50000.0 

101 self.h2["MJD-OBS"] = 49000.0 

102 self.h3["MJD-OBS"] = 53000.0 

103 self.h4["MJD-OBS"] = 52000.0 

104 

105 def test_fail(self): 

106 with self.assertRaises(ValueError): 

107 merge_headers([self.h1, self.h2], mode="wrong") 

108 

109 with self.assertRaises(ValueError): 

110 merge_headers([]) 

111 

112 def test_one(self): 

113 merged = merge_headers([self.h1], mode="drop") 

114 self.assertEqual(merged, self.h1) 

115 

116 def test_merging_overwrite(self): 

117 merged = merge_headers([self.h1, self.h2], mode="overwrite") 

118 # The merged header should be the same type as the first header 

119 self.assertIsInstance(merged, type(self.h1)) 

120 

121 expected = { 

122 "MJD-OBS": self.h2["MJD-OBS"], 

123 "ORIGIN": self.h2["ORIGIN"], 

124 "KEY0": self.h2["KEY0"], 

125 "KEY1": self.h1["KEY1"], 

126 "KEY2": self.h2["KEY2"], 

127 "KEY3": self.h1["KEY3"], 

128 "KEY4": self.h1["KEY4"], 

129 "KEY5": self.h2["KEY5"], 

130 } 

131 self.assertEqual(merged, expected) 

132 

133 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

134 mode="overwrite") 

135 

136 expected = { 

137 "MJD-OBS": self.h4["MJD-OBS"], 

138 "ORIGIN": self.h3["ORIGIN"], 

139 "KEY0": self.h2["KEY0"], 

140 "KEY1": self.h4["KEY1"], 

141 "KEY2": self.h3["KEY2"], 

142 "KEY3": self.h3["KEY3"], 

143 "KEY4": self.h1["KEY4"], 

144 "KEY5": self.h3["KEY5"], 

145 "KEY6": self.h4["KEY6"], 

146 } 

147 

148 self.assertEqual(merged, expected) 

149 

150 def test_merging_first(self): 

151 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

152 mode="first") 

153 

154 expected = { 

155 "MJD-OBS": self.h1["MJD-OBS"], 

156 "ORIGIN": self.h1["ORIGIN"], 

157 "KEY0": self.h1["KEY0"], 

158 "KEY1": self.h1["KEY1"], 

159 "KEY2": self.h1["KEY2"], 

160 "KEY3": self.h1["KEY3"], 

161 "KEY4": self.h1["KEY4"], 

162 "KEY5": self.h2["KEY5"], 

163 "KEY6": self.h4["KEY6"], 

164 } 

165 

166 self.assertEqual(merged, expected) 

167 

168 def test_merging_drop(self): 

169 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

170 mode="drop") 

171 

172 expected = { 

173 "KEY3": self.h1["KEY3"], 

174 "KEY4": self.h1["KEY4"], 

175 "KEY5": self.h2["KEY5"], 

176 "KEY6": self.h4["KEY6"], 

177 } 

178 

179 self.assertEqual(merged, expected) 

180 

181 # Sorting the headers should make no difference to drop mode 

182 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

183 mode="drop", sort=True) 

184 self.assertEqual(merged, expected) 

185 

186 # Now retain some headers 

187 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

188 mode="drop", sort=False, first=["ORIGIN"], last=["KEY2", "KEY1"]) 

189 

190 expected = { 

191 "KEY2": self.h3["KEY2"], 

192 "ORIGIN": self.h1["ORIGIN"], 

193 "KEY1": self.h4["KEY1"], 

194 "KEY3": self.h1["KEY3"], 

195 "KEY4": self.h1["KEY4"], 

196 "KEY5": self.h2["KEY5"], 

197 "KEY6": self.h4["KEY6"], 

198 } 

199 self.assertEqual(merged, expected) 

200 

201 # Now retain some headers with sorting 

202 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

203 mode="drop", sort=True, first=["ORIGIN"], last=["KEY2", "KEY1"]) 

204 

205 expected = { 

206 "KEY2": self.h3["KEY2"], 

207 "ORIGIN": self.h2["ORIGIN"], 

208 "KEY1": self.h4["KEY1"], 

209 "KEY3": self.h1["KEY3"], 

210 "KEY4": self.h1["KEY4"], 

211 "KEY5": self.h2["KEY5"], 

212 "KEY6": self.h4["KEY6"], 

213 } 

214 self.assertEqual(merged, expected) 

215 

216 def test_merging_diff(self): 

217 self.maxDiff = None 

218 

219 # Nothing in common for diff 

220 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

221 mode="diff") 

222 

223 expected = { 

224 "__DIFF__": [self.h1, self.h2, self.h3, self.h4] 

225 } 

226 

227 self.assertEqual(merged, expected) 

228 

229 # Now with a subset that does have overlap 

230 merged = merge_headers([self.h1, self.h2], 

231 mode="diff") 

232 expected = { 

233 "ORIGIN": "LSST", 

234 "__DIFF__": [ 

235 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

236 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

237 ] 

238 } 

239 self.assertEqual(merged, expected) 

240 

241 # Reverse to make sure there is nothing special about the first header 

242 merged = merge_headers([self.h2, self.h1], 

243 mode="diff") 

244 expected = { 

245 "ORIGIN": "LSST", 

246 "__DIFF__": [ 

247 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

248 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

249 ] 

250 } 

251 self.assertEqual(merged, expected) 

252 

253 # Check that identical headers have empty diff 

254 merged = merge_headers([self.h1, self.h1], 

255 mode="diff") 

256 expected = { 

257 **self.h1, 

258 "__DIFF__": [ 

259 {}, 

260 {}, 

261 ] 

262 } 

263 self.assertEqual(merged, expected) 

264 

265 def test_merging_append(self): 

266 # Try with two headers first 

267 merged = merge_headers([self.h1, self.h2], mode="append") 

268 

269 expected = { 

270 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]], 

271 "ORIGIN": self.h1["ORIGIN"], 

272 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]], 

273 "KEY1": self.h1["KEY1"], 

274 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]], 

275 "KEY3": self.h1["KEY3"], 

276 "KEY4": self.h1["KEY4"], 

277 "KEY5": self.h2["KEY5"], 

278 } 

279 

280 self.assertEqual(merged, expected) 

281 

282 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

283 mode="append") 

284 

285 expected = { 

286 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]], 

287 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None], 

288 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None], 

289 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]], 

290 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None], 

291 "KEY3": self.h3["KEY3"], 

292 "KEY4": self.h1["KEY4"], 

293 "KEY5": self.h3["KEY5"], 

294 "KEY6": self.h4["KEY6"], 

295 } 

296 

297 self.assertEqual(merged, expected) 

298 

299 def test_merging_overwrite_sort(self): 

300 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True) 

301 

302 expected = { 

303 "MJD-OBS": self.h1["MJD-OBS"], 

304 "ORIGIN": self.h1["ORIGIN"], 

305 "KEY0": self.h1["KEY0"], 

306 "KEY1": self.h1["KEY1"], 

307 "KEY2": self.h1["KEY2"], 

308 "KEY3": self.h1["KEY3"], 

309 "KEY4": self.h1["KEY4"], 

310 "KEY5": self.h2["KEY5"], 

311 } 

312 self.assertEqual(merged, expected) 

313 

314 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

315 mode="overwrite", sort=True) 

316 

317 expected = { 

318 "MJD-OBS": self.h3["MJD-OBS"], 

319 "ORIGIN": self.h3["ORIGIN"], 

320 "KEY0": self.h1["KEY0"], 

321 "KEY1": self.h4["KEY1"], 

322 "KEY2": self.h3["KEY2"], 

323 "KEY3": self.h3["KEY3"], 

324 "KEY4": self.h1["KEY4"], 

325 "KEY5": self.h3["KEY5"], 

326 "KEY6": self.h4["KEY6"], 

327 } 

328 

329 self.assertEqual(merged, expected) 

330 

331 # Changing the order should not change the result 

332 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], 

333 mode="overwrite", sort=True) 

334 

335 self.assertEqual(merged, expected) 

336 

337 def test_merging_first_sort(self): 

338 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

339 mode="first", sort=True) 

340 

341 expected = { 

342 "MJD-OBS": self.h2["MJD-OBS"], 

343 "ORIGIN": self.h2["ORIGIN"], 

344 "KEY0": self.h2["KEY0"], 

345 "KEY1": self.h1["KEY1"], 

346 "KEY2": self.h2["KEY2"], 

347 "KEY3": self.h1["KEY3"], 

348 "KEY4": self.h1["KEY4"], 

349 "KEY5": self.h2["KEY5"], 

350 "KEY6": self.h4["KEY6"], 

351 } 

352 

353 self.assertEqual(merged, expected) 

354 

355 def test_merging_append_sort(self): 

356 # Try with two headers first 

357 merged = merge_headers([self.h1, self.h2], mode="append", sort=True) 

358 

359 expected = { 

360 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]], 

361 "ORIGIN": self.h1["ORIGIN"], 

362 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]], 

363 "KEY1": self.h1["KEY1"], 

364 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]], 

365 "KEY3": self.h1["KEY3"], 

366 "KEY4": self.h1["KEY4"], 

367 "KEY5": self.h2["KEY5"], 

368 } 

369 

370 self.assertEqual(merged, expected) 

371 

372 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

373 mode="append", sort=True) 

374 

375 expected = { 

376 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]], 

377 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]], 

378 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None], 

379 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None], 

380 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]], 

381 "KEY3": self.h3["KEY3"], 

382 "KEY4": self.h1["KEY4"], 

383 "KEY5": self.h3["KEY5"], 

384 "KEY6": self.h4["KEY6"], 

385 } 

386 

387 self.assertEqual(merged, expected) 

388 

389 # Order should not matter 

390 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], 

391 mode="append", sort=True) 

392 self.assertEqual(merged, expected) 

393 

394 

395class FixHeadersTestCase(unittest.TestCase): 

396 

397 def test_basic_fix_header(self): 

398 """Test that a header can be fixed if we specify a local path. 

399 """ 

400 

401 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

402 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3") 

403 

404 # First fix header but using no search path (should work as no-op) 

405 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

406 self.assertFalse(fixed) 

407 

408 # Now using the test corrections directory 

409 header2 = copy.copy(header) 

410 fixed = fix_header(header2, search_path=os.path.join(TESTDIR, "data", "corrections"), 

411 translator_class=NullDecamTranslator) 

412 self.assertTrue(fixed) 

413 self.assertEqual(header2["DETECTOR"], "NEW-ID") 

414 

415 # Now with a corrections directory that has bad YAML in it 

416 header2 = copy.copy(header) 

417 with self.assertLogs(level="WARN"): 

418 fixed = fix_header(header2, search_path=os.path.join(TESTDIR, "data", "bad_corrections"), 

419 translator_class=NullDecamTranslator) 

420 self.assertFalse(fixed) 

421 

422 # Test that fix_header of unknown header is allowed 

423 header = {"SOMETHING": "UNKNOWN"} 

424 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

425 self.assertFalse(fixed) 

426 

427 def test_hsc_fix_header(self): 

428 """Check that one of the known HSC corrections is being applied 

429 properly.""" 

430 header = {"EXP-ID": "HSCA00120800", 

431 "INSTRUME": "HSC", 

432 "DATA-TYP": "FLAT"} 

433 

434 fixed = fix_header(header, translator_class=HscTranslator) 

435 self.assertTrue(fixed) 

436 self.assertEqual(header["DATA-TYP"], "OBJECT") 

437 

438 # Check provenance 

439 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"]) 

440 

441 # And that this header won't be corrected 

442 header = {"EXP-ID": "HSCA00120800X", 

443 "INSTRUME": "HSC", 

444 "DATA-TYP": "FLAT"} 

445 

446 fixed = fix_header(header, translator_class=HscTranslator) 

447 self.assertFalse(fixed) 

448 self.assertEqual(header["DATA-TYP"], "FLAT") 

449 

450 def test_decam_fix_header(self): 

451 """Check that one of the known DECam corrections is being applied 

452 properly.""" 

453 

454 # This header is a bias (zero) with an erroneous Y filter 

455 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

456 fixed = fix_header(header, translator_class=DecamTranslator) 

457 self.assertTrue(fixed) 

458 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0") 

459 

460 def test_translator_fix_header(self): 

461 """Check that translator classes can fix headers.""" 

462 

463 # Read in a known header 

464 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

465 self.assertEqual(header["DTSITE"], "ct") 

466 

467 header2 = copy.copy(header) 

468 fixed = fix_header(header2, translator_class=NotDecamTranslator) 

469 self.assertTrue(fixed) 

470 self.assertEqual(header2["DTSITE"], "hi") 

471 

472 header2 = copy.copy(header) 

473 header2["DTSITE"] = "reset" 

474 with self.assertLogs("astro_metadata_translator", level="FATAL"): 

475 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator) 

476 self.assertFalse(fixed) 

477 self.assertEqual(header2["DTSITE"], "reset") 

478 

479 def test_no_double_fix(self): 

480 """Check that header fixup only happens once.""" 

481 

482 # Read in a known header 

483 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

484 self.assertEqual(header["DTSITE"], "ct") 

485 

486 # First time it will modifiy DTSITE 

487 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

488 self.assertTrue(fixed) 

489 self.assertEqual(header["DTSITE"], "cthi") 

490 

491 # Get the fix up date 

492 date = header["HIERARCH ASTRO METADATA FIX DATE"] 

493 

494 # Second time it will do nothing but still report it was fixed 

495 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

496 self.assertTrue(fixed) 

497 self.assertEqual(header["DTSITE"], "cthi") 

498 

499 # Date of fixup should be the same 

500 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date) 

501 

502 # Test the translator version in provenance 

503 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0") 

504 

505 

506if __name__ == "__main__": 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true

507 unittest.main()