Coverage for tests/test_headers.py: 27%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 10:02 +0000

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import os.path 

14import unittest 

15 

16from astro_metadata_translator import DecamTranslator, HscTranslator, fix_header, merge_headers 

17from astro_metadata_translator.tests import read_test_file 

18 

19TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

20 

21 

22class NotDecamTranslator(DecamTranslator): 

23 """A DECam translator with override list of header corrections.""" 

24 

25 name = None 

26 

27 @classmethod 

28 def fix_header(cls, header, instrument, obsid, filename=None): 

29 header["DTSITE"] = "hi" 

30 return True 

31 

32 @classmethod 

33 def translator_version(cls): 

34 # Hardcode a version so we can test for it 

35 return "1.0.0" 

36 

37 

38class NotDecamTranslator2(NotDecamTranslator): 

39 """Similar to NotDecamTranslator but has a fixup that will break on 

40 repeat. 

41 """ 

42 

43 name = None 

44 

45 @classmethod 

46 def fix_header(cls, header, instrument, obsid, filename=None): 

47 header["DTSITE"] += "hi" 

48 return True 

49 

50 

51class AlsoNotDecamTranslator(DecamTranslator): 

52 """A DECam translator with override list of header corrections 

53 that fails. 

54 """ 

55 

56 name = None 

57 

58 @classmethod 

59 def fix_header(cls, header, instrument, obsid, filename=None): 

60 raise RuntimeError("Failure to work something out from header") 

61 

62 

63class NullDecamTranslator(DecamTranslator): 

64 """A DECam translator that doesn't do any fixes.""" 

65 

66 name = None 

67 

68 @classmethod 

69 def fix_header(cls, header, instrument, obsid, filename=None): 

70 return False 

71 

72 

73class HeadersTestCase(unittest.TestCase): 

74 """Test header manipulation utilities.""" 

75 

76 def setUp(self): 

77 # Define reference headers 

78 self.h1 = dict( 

79 ORIGIN="LSST", 

80 KEY0=0, 

81 KEY1=1, 

82 KEY2=3, 

83 KEY3=3.1415, 

84 KEY4="a", 

85 ) 

86 

87 self.h2 = dict(ORIGIN="LSST", KEY0="0", KEY2=4, KEY5=42) 

88 self.h3 = dict( 

89 ORIGIN="AUXTEL", 

90 KEY3=3.1415, 

91 KEY2=50, 

92 KEY5=42, 

93 ) 

94 self.h4 = dict( 

95 KEY6="New", 

96 KEY1="Exists", 

97 ) 

98 

99 # Add keys for sorting by time 

100 # Sorted order: h2, h1, h4, h3 

101 self.h1["MJD-OBS"] = 50000.0 

102 self.h2["MJD-OBS"] = 49000.0 

103 self.h3["MJD-OBS"] = 53000.0 

104 self.h4["MJD-OBS"] = 52000.0 

105 

106 def test_fail(self): 

107 with self.assertRaises(ValueError): 

108 merge_headers([self.h1, self.h2], mode="wrong") 

109 

110 with self.assertRaises(ValueError): 

111 merge_headers([]) 

112 

113 def test_one(self): 

114 merged = merge_headers([self.h1], mode="drop") 

115 self.assertEqual(merged, self.h1) 

116 

117 def test_merging_overwrite(self): 

118 merged = merge_headers([self.h1, self.h2], mode="overwrite") 

119 # The merged header should be the same type as the first header 

120 self.assertIsInstance(merged, type(self.h1)) 

121 

122 expected = { 

123 "MJD-OBS": self.h2["MJD-OBS"], 

124 "ORIGIN": self.h2["ORIGIN"], 

125 "KEY0": self.h2["KEY0"], 

126 "KEY1": self.h1["KEY1"], 

127 "KEY2": self.h2["KEY2"], 

128 "KEY3": self.h1["KEY3"], 

129 "KEY4": self.h1["KEY4"], 

130 "KEY5": self.h2["KEY5"], 

131 } 

132 self.assertEqual(merged, expected) 

133 

134 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite") 

135 

136 expected = { 

137 "MJD-OBS": self.h4["MJD-OBS"], 

138 "ORIGIN": self.h3["ORIGIN"], 

139 "KEY0": self.h2["KEY0"], 

140 "KEY1": self.h4["KEY1"], 

141 "KEY2": self.h3["KEY2"], 

142 "KEY3": self.h3["KEY3"], 

143 "KEY4": self.h1["KEY4"], 

144 "KEY5": self.h3["KEY5"], 

145 "KEY6": self.h4["KEY6"], 

146 } 

147 

148 self.assertEqual(merged, expected) 

149 

150 def test_merging_first(self): 

151 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first") 

152 

153 expected = { 

154 "MJD-OBS": self.h1["MJD-OBS"], 

155 "ORIGIN": self.h1["ORIGIN"], 

156 "KEY0": self.h1["KEY0"], 

157 "KEY1": self.h1["KEY1"], 

158 "KEY2": self.h1["KEY2"], 

159 "KEY3": self.h1["KEY3"], 

160 "KEY4": self.h1["KEY4"], 

161 "KEY5": self.h2["KEY5"], 

162 "KEY6": self.h4["KEY6"], 

163 } 

164 

165 self.assertEqual(merged, expected) 

166 

167 def test_merging_drop(self): 

168 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop") 

169 

170 expected = { 

171 "KEY3": self.h1["KEY3"], 

172 "KEY4": self.h1["KEY4"], 

173 "KEY5": self.h2["KEY5"], 

174 "KEY6": self.h4["KEY6"], 

175 } 

176 

177 self.assertEqual(merged, expected) 

178 

179 # Sorting the headers should make no difference to drop mode 

180 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop", sort=True) 

181 self.assertEqual(merged, expected) 

182 

183 # Now retain some headers 

184 merged = merge_headers( 

185 [self.h1, self.h2, self.h3, self.h4], 

186 mode="drop", 

187 sort=False, 

188 first=["ORIGIN"], 

189 last=["KEY2", "KEY1"], 

190 ) 

191 

192 expected = { 

193 "KEY2": self.h3["KEY2"], 

194 "ORIGIN": self.h1["ORIGIN"], 

195 "KEY1": self.h4["KEY1"], 

196 "KEY3": self.h1["KEY3"], 

197 "KEY4": self.h1["KEY4"], 

198 "KEY5": self.h2["KEY5"], 

199 "KEY6": self.h4["KEY6"], 

200 } 

201 self.assertEqual(merged, expected) 

202 

203 # Now retain some headers with sorting 

204 merged = merge_headers( 

205 [self.h1, self.h2, self.h3, self.h4], 

206 mode="drop", 

207 sort=True, 

208 first=["ORIGIN"], 

209 last=["KEY2", "KEY1"], 

210 ) 

211 

212 expected = { 

213 "KEY2": self.h3["KEY2"], 

214 "ORIGIN": self.h2["ORIGIN"], 

215 "KEY1": self.h4["KEY1"], 

216 "KEY3": self.h1["KEY3"], 

217 "KEY4": self.h1["KEY4"], 

218 "KEY5": self.h2["KEY5"], 

219 "KEY6": self.h4["KEY6"], 

220 } 

221 self.assertEqual(merged, expected) 

222 

223 def test_merging_diff(self): 

224 self.maxDiff = None 

225 

226 # Nothing in common for diff 

227 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="diff") 

228 

229 expected = {"__DIFF__": [self.h1, self.h2, self.h3, self.h4]} 

230 

231 self.assertEqual(merged, expected) 

232 

233 # Now with a subset that does have overlap 

234 merged = merge_headers([self.h1, self.h2], mode="diff") 

235 expected = { 

236 "ORIGIN": "LSST", 

237 "__DIFF__": [ 

238 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

239 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

240 ], 

241 } 

242 self.assertEqual(merged, expected) 

243 

244 # Reverse to make sure there is nothing special about the first header 

245 merged = merge_headers([self.h2, self.h1], mode="diff") 

246 expected = { 

247 "ORIGIN": "LSST", 

248 "__DIFF__": [ 

249 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

250 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

251 ], 

252 } 

253 self.assertEqual(merged, expected) 

254 

255 # Check that identical headers have empty diff 

256 merged = merge_headers([self.h1, self.h1], mode="diff") 

257 expected = { 

258 **self.h1, 

259 "__DIFF__": [ 

260 {}, 

261 {}, 

262 ], 

263 } 

264 self.assertEqual(merged, expected) 

265 

266 def test_merging_append(self): 

267 # Try with two headers first 

268 merged = merge_headers([self.h1, self.h2], mode="append") 

269 

270 expected = { 

271 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]], 

272 "ORIGIN": self.h1["ORIGIN"], 

273 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]], 

274 "KEY1": self.h1["KEY1"], 

275 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]], 

276 "KEY3": self.h1["KEY3"], 

277 "KEY4": self.h1["KEY4"], 

278 "KEY5": self.h2["KEY5"], 

279 } 

280 

281 self.assertEqual(merged, expected) 

282 

283 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append") 

284 

285 expected = { 

286 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]], 

287 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None], 

288 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None], 

289 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]], 

290 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None], 

291 "KEY3": self.h3["KEY3"], 

292 "KEY4": self.h1["KEY4"], 

293 "KEY5": self.h3["KEY5"], 

294 "KEY6": self.h4["KEY6"], 

295 } 

296 

297 self.assertEqual(merged, expected) 

298 

299 def test_merging_overwrite_sort(self): 

300 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True) 

301 

302 expected = { 

303 "MJD-OBS": self.h1["MJD-OBS"], 

304 "ORIGIN": self.h1["ORIGIN"], 

305 "KEY0": self.h1["KEY0"], 

306 "KEY1": self.h1["KEY1"], 

307 "KEY2": self.h1["KEY2"], 

308 "KEY3": self.h1["KEY3"], 

309 "KEY4": self.h1["KEY4"], 

310 "KEY5": self.h2["KEY5"], 

311 } 

312 self.assertEqual(merged, expected) 

313 

314 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite", sort=True) 

315 

316 expected = { 

317 "MJD-OBS": self.h3["MJD-OBS"], 

318 "ORIGIN": self.h3["ORIGIN"], 

319 "KEY0": self.h1["KEY0"], 

320 "KEY1": self.h4["KEY1"], 

321 "KEY2": self.h3["KEY2"], 

322 "KEY3": self.h3["KEY3"], 

323 "KEY4": self.h1["KEY4"], 

324 "KEY5": self.h3["KEY5"], 

325 "KEY6": self.h4["KEY6"], 

326 } 

327 

328 self.assertEqual(merged, expected) 

329 

330 # Changing the order should not change the result 

331 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], mode="overwrite", sort=True) 

332 

333 self.assertEqual(merged, expected) 

334 

335 def test_merging_first_sort(self): 

336 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first", sort=True) 

337 

338 expected = { 

339 "MJD-OBS": self.h2["MJD-OBS"], 

340 "ORIGIN": self.h2["ORIGIN"], 

341 "KEY0": self.h2["KEY0"], 

342 "KEY1": self.h1["KEY1"], 

343 "KEY2": self.h2["KEY2"], 

344 "KEY3": self.h1["KEY3"], 

345 "KEY4": self.h1["KEY4"], 

346 "KEY5": self.h2["KEY5"], 

347 "KEY6": self.h4["KEY6"], 

348 } 

349 

350 self.assertEqual(merged, expected) 

351 

352 def test_merging_append_sort(self): 

353 # Try with two headers first 

354 merged = merge_headers([self.h1, self.h2], mode="append", sort=True) 

355 

356 expected = { 

357 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]], 

358 "ORIGIN": self.h1["ORIGIN"], 

359 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]], 

360 "KEY1": self.h1["KEY1"], 

361 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]], 

362 "KEY3": self.h1["KEY3"], 

363 "KEY4": self.h1["KEY4"], 

364 "KEY5": self.h2["KEY5"], 

365 } 

366 

367 self.assertEqual(merged, expected) 

368 

369 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append", sort=True) 

370 

371 expected = { 

372 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]], 

373 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]], 

374 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None], 

375 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None], 

376 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]], 

377 "KEY3": self.h3["KEY3"], 

378 "KEY4": self.h1["KEY4"], 

379 "KEY5": self.h3["KEY5"], 

380 "KEY6": self.h4["KEY6"], 

381 } 

382 

383 self.assertEqual(merged, expected) 

384 

385 # Order should not matter 

386 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], mode="append", sort=True) 

387 self.assertEqual(merged, expected) 

388 

389 

390class FixHeadersTestCase(unittest.TestCase): 

391 """Test header fix up.""" 

392 

393 def test_basic_fix_header(self): 

394 """Test that a header can be fixed if we specify a local path.""" 

395 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

396 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3") 

397 

398 # First fix header but using no search path (should work as no-op) 

399 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

400 self.assertFalse(fixed) 

401 

402 # Now using the test corrections directory 

403 header2 = copy.copy(header) 

404 fixed = fix_header( 

405 header2, 

406 search_path=os.path.join(TESTDIR, "data", "corrections"), 

407 translator_class=NullDecamTranslator, 

408 ) 

409 self.assertTrue(fixed) 

410 self.assertEqual(header2["DETECTOR"], "NEW-ID") 

411 

412 # Now with a corrections directory that has bad YAML in it 

413 header2 = copy.copy(header) 

414 with self.assertLogs(level="WARN"): 

415 fixed = fix_header( 

416 header2, 

417 search_path=os.path.join(TESTDIR, "data", "bad_corrections"), 

418 translator_class=NullDecamTranslator, 

419 ) 

420 self.assertFalse(fixed) 

421 

422 # Test that fix_header of unknown header is allowed 

423 header = {"SOMETHING": "UNKNOWN"} 

424 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

425 self.assertFalse(fixed) 

426 

427 def test_hsc_fix_header(self): 

428 """Check that one of the known HSC corrections is being applied 

429 properly. 

430 """ 

431 header = {"EXP-ID": "HSCA00120800", "INSTRUME": "HSC", "DATA-TYP": "FLAT"} 

432 

433 fixed = fix_header(header, translator_class=HscTranslator) 

434 self.assertTrue(fixed) 

435 self.assertEqual(header["DATA-TYP"], "OBJECT") 

436 

437 # Check provenance 

438 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"]) 

439 

440 # And that this header won't be corrected 

441 header = {"EXP-ID": "HSCA00120800X", "INSTRUME": "HSC", "DATA-TYP": "FLAT"} 

442 

443 fixed = fix_header(header, translator_class=HscTranslator) 

444 self.assertFalse(fixed) 

445 self.assertEqual(header["DATA-TYP"], "FLAT") 

446 

447 def test_decam_fix_header(self): 

448 """Check that one of the known DECam corrections is being applied 

449 properly. 

450 """ 

451 # This header is a bias (zero) with an erroneous Y filter 

452 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

453 fixed = fix_header(header, translator_class=DecamTranslator) 

454 self.assertTrue(fixed) 

455 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0") 

456 

457 def test_translator_fix_header(self): 

458 """Check that translator classes can fix headers.""" 

459 # Read in a known header 

460 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

461 self.assertEqual(header["DTSITE"], "ct") 

462 

463 header2 = copy.copy(header) 

464 fixed = fix_header(header2, translator_class=NotDecamTranslator) 

465 self.assertTrue(fixed) 

466 self.assertEqual(header2["DTSITE"], "hi") 

467 

468 header2 = copy.copy(header) 

469 header2["DTSITE"] = "reset" 

470 with self.assertLogs("astro_metadata_translator", level="FATAL"): 

471 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator) 

472 self.assertFalse(fixed) 

473 self.assertEqual(header2["DTSITE"], "reset") 

474 

475 def test_no_double_fix(self): 

476 """Check that header fixup only happens once.""" 

477 # Read in a known header 

478 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

479 self.assertEqual(header["DTSITE"], "ct") 

480 

481 # First time it will modifiy DTSITE 

482 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

483 self.assertTrue(fixed) 

484 self.assertEqual(header["DTSITE"], "cthi") 

485 

486 # Get the fix up date 

487 date = header["HIERARCH ASTRO METADATA FIX DATE"] 

488 

489 # Second time it will do nothing but still report it was fixed 

490 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

491 self.assertTrue(fixed) 

492 self.assertEqual(header["DTSITE"], "cthi") 

493 

494 # Date of fixup should be the same 

495 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date) 

496 

497 # Test the translator version in provenance 

498 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0") 

499 

500 

501if __name__ == "__main__": 501 ↛ 502line 501 didn't jump to line 502, because the condition on line 501 was never true

502 unittest.main()