Coverage for tests/test_headers.py: 23%

175 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-30 02:35 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import os.path 

14import unittest 

15 

16from astro_metadata_translator import DecamTranslator, HscTranslator, fix_header, merge_headers 

17from astro_metadata_translator.tests import read_test_file 

18 

19TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

20 

21 

22class NotDecamTranslator(DecamTranslator): 

23 """This is a DECam translator with override list of header corrections.""" 

24 

25 name = None 

26 

27 @classmethod 

28 def fix_header(cls, header, instrument, obsid, filename=None): 

29 header["DTSITE"] = "hi" 

30 return True 

31 

32 @classmethod 

33 def translator_version(cls): 

34 # Hardcode a version so we can test for it 

35 return "1.0.0" 

36 

37 

38class NotDecamTranslator2(NotDecamTranslator): 

39 """This is like NotDecamTranslator but has a fixup that will break on 

40 repeat.""" 

41 

42 name = None 

43 

44 @classmethod 

45 def fix_header(cls, header, instrument, obsid, filename=None): 

46 header["DTSITE"] += "hi" 

47 return True 

48 

49 

50class AlsoNotDecamTranslator(DecamTranslator): 

51 """This is a DECam translator with override list of header corrections 

52 that fails.""" 

53 

54 name = None 

55 

56 @classmethod 

57 def fix_header(cls, header, instrument, obsid, filename=None): 

58 raise RuntimeError("Failure to work something out from header") 

59 

60 

61class NullDecamTranslator(DecamTranslator): 

62 """This is a DECam translator that doesn't do any fixes.""" 

63 

64 name = None 

65 

66 @classmethod 

67 def fix_header(cls, header, instrument, obsid, filename=None): 

68 return False 

69 

70 

71class HeadersTestCase(unittest.TestCase): 

72 def setUp(self): 

73 # Define reference headers 

74 self.h1 = dict( 

75 ORIGIN="LSST", 

76 KEY0=0, 

77 KEY1=1, 

78 KEY2=3, 

79 KEY3=3.1415, 

80 KEY4="a", 

81 ) 

82 

83 self.h2 = dict(ORIGIN="LSST", KEY0="0", KEY2=4, KEY5=42) 

84 self.h3 = dict( 

85 ORIGIN="AUXTEL", 

86 KEY3=3.1415, 

87 KEY2=50, 

88 KEY5=42, 

89 ) 

90 self.h4 = dict( 

91 KEY6="New", 

92 KEY1="Exists", 

93 ) 

94 

95 # Add keys for sorting by time 

96 # Sorted order: h2, h1, h4, h3 

97 self.h1["MJD-OBS"] = 50000.0 

98 self.h2["MJD-OBS"] = 49000.0 

99 self.h3["MJD-OBS"] = 53000.0 

100 self.h4["MJD-OBS"] = 52000.0 

101 

102 def test_fail(self): 

103 with self.assertRaises(ValueError): 

104 merge_headers([self.h1, self.h2], mode="wrong") 

105 

106 with self.assertRaises(ValueError): 

107 merge_headers([]) 

108 

109 def test_one(self): 

110 merged = merge_headers([self.h1], mode="drop") 

111 self.assertEqual(merged, self.h1) 

112 

113 def test_merging_overwrite(self): 

114 merged = merge_headers([self.h1, self.h2], mode="overwrite") 

115 # The merged header should be the same type as the first header 

116 self.assertIsInstance(merged, type(self.h1)) 

117 

118 expected = { 

119 "MJD-OBS": self.h2["MJD-OBS"], 

120 "ORIGIN": self.h2["ORIGIN"], 

121 "KEY0": self.h2["KEY0"], 

122 "KEY1": self.h1["KEY1"], 

123 "KEY2": self.h2["KEY2"], 

124 "KEY3": self.h1["KEY3"], 

125 "KEY4": self.h1["KEY4"], 

126 "KEY5": self.h2["KEY5"], 

127 } 

128 self.assertEqual(merged, expected) 

129 

130 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite") 

131 

132 expected = { 

133 "MJD-OBS": self.h4["MJD-OBS"], 

134 "ORIGIN": self.h3["ORIGIN"], 

135 "KEY0": self.h2["KEY0"], 

136 "KEY1": self.h4["KEY1"], 

137 "KEY2": self.h3["KEY2"], 

138 "KEY3": self.h3["KEY3"], 

139 "KEY4": self.h1["KEY4"], 

140 "KEY5": self.h3["KEY5"], 

141 "KEY6": self.h4["KEY6"], 

142 } 

143 

144 self.assertEqual(merged, expected) 

145 

146 def test_merging_first(self): 

147 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first") 

148 

149 expected = { 

150 "MJD-OBS": self.h1["MJD-OBS"], 

151 "ORIGIN": self.h1["ORIGIN"], 

152 "KEY0": self.h1["KEY0"], 

153 "KEY1": self.h1["KEY1"], 

154 "KEY2": self.h1["KEY2"], 

155 "KEY3": self.h1["KEY3"], 

156 "KEY4": self.h1["KEY4"], 

157 "KEY5": self.h2["KEY5"], 

158 "KEY6": self.h4["KEY6"], 

159 } 

160 

161 self.assertEqual(merged, expected) 

162 

163 def test_merging_drop(self): 

164 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop") 

165 

166 expected = { 

167 "KEY3": self.h1["KEY3"], 

168 "KEY4": self.h1["KEY4"], 

169 "KEY5": self.h2["KEY5"], 

170 "KEY6": self.h4["KEY6"], 

171 } 

172 

173 self.assertEqual(merged, expected) 

174 

175 # Sorting the headers should make no difference to drop mode 

176 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="drop", sort=True) 

177 self.assertEqual(merged, expected) 

178 

179 # Now retain some headers 

180 merged = merge_headers( 

181 [self.h1, self.h2, self.h3, self.h4], 

182 mode="drop", 

183 sort=False, 

184 first=["ORIGIN"], 

185 last=["KEY2", "KEY1"], 

186 ) 

187 

188 expected = { 

189 "KEY2": self.h3["KEY2"], 

190 "ORIGIN": self.h1["ORIGIN"], 

191 "KEY1": self.h4["KEY1"], 

192 "KEY3": self.h1["KEY3"], 

193 "KEY4": self.h1["KEY4"], 

194 "KEY5": self.h2["KEY5"], 

195 "KEY6": self.h4["KEY6"], 

196 } 

197 self.assertEqual(merged, expected) 

198 

199 # Now retain some headers with sorting 

200 merged = merge_headers( 

201 [self.h1, self.h2, self.h3, self.h4], 

202 mode="drop", 

203 sort=True, 

204 first=["ORIGIN"], 

205 last=["KEY2", "KEY1"], 

206 ) 

207 

208 expected = { 

209 "KEY2": self.h3["KEY2"], 

210 "ORIGIN": self.h2["ORIGIN"], 

211 "KEY1": self.h4["KEY1"], 

212 "KEY3": self.h1["KEY3"], 

213 "KEY4": self.h1["KEY4"], 

214 "KEY5": self.h2["KEY5"], 

215 "KEY6": self.h4["KEY6"], 

216 } 

217 self.assertEqual(merged, expected) 

218 

219 def test_merging_diff(self): 

220 self.maxDiff = None 

221 

222 # Nothing in common for diff 

223 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="diff") 

224 

225 expected = {"__DIFF__": [self.h1, self.h2, self.h3, self.h4]} 

226 

227 self.assertEqual(merged, expected) 

228 

229 # Now with a subset that does have overlap 

230 merged = merge_headers([self.h1, self.h2], mode="diff") 

231 expected = { 

232 "ORIGIN": "LSST", 

233 "__DIFF__": [ 

234 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

235 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

236 ], 

237 } 

238 self.assertEqual(merged, expected) 

239 

240 # Reverse to make sure there is nothing special about the first header 

241 merged = merge_headers([self.h2, self.h1], mode="diff") 

242 expected = { 

243 "ORIGIN": "LSST", 

244 "__DIFF__": [ 

245 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

246 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

247 ], 

248 } 

249 self.assertEqual(merged, expected) 

250 

251 # Check that identical headers have empty diff 

252 merged = merge_headers([self.h1, self.h1], mode="diff") 

253 expected = { 

254 **self.h1, 

255 "__DIFF__": [ 

256 {}, 

257 {}, 

258 ], 

259 } 

260 self.assertEqual(merged, expected) 

261 

262 def test_merging_append(self): 

263 # Try with two headers first 

264 merged = merge_headers([self.h1, self.h2], mode="append") 

265 

266 expected = { 

267 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]], 

268 "ORIGIN": self.h1["ORIGIN"], 

269 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]], 

270 "KEY1": self.h1["KEY1"], 

271 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]], 

272 "KEY3": self.h1["KEY3"], 

273 "KEY4": self.h1["KEY4"], 

274 "KEY5": self.h2["KEY5"], 

275 } 

276 

277 self.assertEqual(merged, expected) 

278 

279 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append") 

280 

281 expected = { 

282 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]], 

283 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None], 

284 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None], 

285 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]], 

286 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None], 

287 "KEY3": self.h3["KEY3"], 

288 "KEY4": self.h1["KEY4"], 

289 "KEY5": self.h3["KEY5"], 

290 "KEY6": self.h4["KEY6"], 

291 } 

292 

293 self.assertEqual(merged, expected) 

294 

295 def test_merging_overwrite_sort(self): 

296 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True) 

297 

298 expected = { 

299 "MJD-OBS": self.h1["MJD-OBS"], 

300 "ORIGIN": self.h1["ORIGIN"], 

301 "KEY0": self.h1["KEY0"], 

302 "KEY1": self.h1["KEY1"], 

303 "KEY2": self.h1["KEY2"], 

304 "KEY3": self.h1["KEY3"], 

305 "KEY4": self.h1["KEY4"], 

306 "KEY5": self.h2["KEY5"], 

307 } 

308 self.assertEqual(merged, expected) 

309 

310 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="overwrite", sort=True) 

311 

312 expected = { 

313 "MJD-OBS": self.h3["MJD-OBS"], 

314 "ORIGIN": self.h3["ORIGIN"], 

315 "KEY0": self.h1["KEY0"], 

316 "KEY1": self.h4["KEY1"], 

317 "KEY2": self.h3["KEY2"], 

318 "KEY3": self.h3["KEY3"], 

319 "KEY4": self.h1["KEY4"], 

320 "KEY5": self.h3["KEY5"], 

321 "KEY6": self.h4["KEY6"], 

322 } 

323 

324 self.assertEqual(merged, expected) 

325 

326 # Changing the order should not change the result 

327 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], mode="overwrite", sort=True) 

328 

329 self.assertEqual(merged, expected) 

330 

331 def test_merging_first_sort(self): 

332 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="first", sort=True) 

333 

334 expected = { 

335 "MJD-OBS": self.h2["MJD-OBS"], 

336 "ORIGIN": self.h2["ORIGIN"], 

337 "KEY0": self.h2["KEY0"], 

338 "KEY1": self.h1["KEY1"], 

339 "KEY2": self.h2["KEY2"], 

340 "KEY3": self.h1["KEY3"], 

341 "KEY4": self.h1["KEY4"], 

342 "KEY5": self.h2["KEY5"], 

343 "KEY6": self.h4["KEY6"], 

344 } 

345 

346 self.assertEqual(merged, expected) 

347 

348 def test_merging_append_sort(self): 

349 # Try with two headers first 

350 merged = merge_headers([self.h1, self.h2], mode="append", sort=True) 

351 

352 expected = { 

353 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]], 

354 "ORIGIN": self.h1["ORIGIN"], 

355 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]], 

356 "KEY1": self.h1["KEY1"], 

357 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]], 

358 "KEY3": self.h1["KEY3"], 

359 "KEY4": self.h1["KEY4"], 

360 "KEY5": self.h2["KEY5"], 

361 } 

362 

363 self.assertEqual(merged, expected) 

364 

365 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], mode="append", sort=True) 

366 

367 expected = { 

368 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]], 

369 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]], 

370 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None], 

371 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None], 

372 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]], 

373 "KEY3": self.h3["KEY3"], 

374 "KEY4": self.h1["KEY4"], 

375 "KEY5": self.h3["KEY5"], 

376 "KEY6": self.h4["KEY6"], 

377 } 

378 

379 self.assertEqual(merged, expected) 

380 

381 # Order should not matter 

382 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], mode="append", sort=True) 

383 self.assertEqual(merged, expected) 

384 

385 

386class FixHeadersTestCase(unittest.TestCase): 

387 def test_basic_fix_header(self): 

388 """Test that a header can be fixed if we specify a local path.""" 

389 

390 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

391 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3") 

392 

393 # First fix header but using no search path (should work as no-op) 

394 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

395 self.assertFalse(fixed) 

396 

397 # Now using the test corrections directory 

398 header2 = copy.copy(header) 

399 fixed = fix_header( 

400 header2, 

401 search_path=os.path.join(TESTDIR, "data", "corrections"), 

402 translator_class=NullDecamTranslator, 

403 ) 

404 self.assertTrue(fixed) 

405 self.assertEqual(header2["DETECTOR"], "NEW-ID") 

406 

407 # Now with a corrections directory that has bad YAML in it 

408 header2 = copy.copy(header) 

409 with self.assertLogs(level="WARN"): 

410 fixed = fix_header( 

411 header2, 

412 search_path=os.path.join(TESTDIR, "data", "bad_corrections"), 

413 translator_class=NullDecamTranslator, 

414 ) 

415 self.assertFalse(fixed) 

416 

417 # Test that fix_header of unknown header is allowed 

418 header = {"SOMETHING": "UNKNOWN"} 

419 fixed = fix_header(copy.copy(header), translator_class=NullDecamTranslator) 

420 self.assertFalse(fixed) 

421 

422 def test_hsc_fix_header(self): 

423 """Check that one of the known HSC corrections is being applied 

424 properly.""" 

425 header = {"EXP-ID": "HSCA00120800", "INSTRUME": "HSC", "DATA-TYP": "FLAT"} 

426 

427 fixed = fix_header(header, translator_class=HscTranslator) 

428 self.assertTrue(fixed) 

429 self.assertEqual(header["DATA-TYP"], "OBJECT") 

430 

431 # Check provenance 

432 self.assertIn("HSC-HSCA00120800.yaml", header["HIERARCH ASTRO METADATA FIX FILE"]) 

433 

434 # And that this header won't be corrected 

435 header = {"EXP-ID": "HSCA00120800X", "INSTRUME": "HSC", "DATA-TYP": "FLAT"} 

436 

437 fixed = fix_header(header, translator_class=HscTranslator) 

438 self.assertFalse(fixed) 

439 self.assertEqual(header["DATA-TYP"], "FLAT") 

440 

441 def test_decam_fix_header(self): 

442 """Check that one of the known DECam corrections is being applied 

443 properly.""" 

444 

445 # This header is a bias (zero) with an erroneous Y filter 

446 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

447 fixed = fix_header(header, translator_class=DecamTranslator) 

448 self.assertTrue(fixed) 

449 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0") 

450 

451 def test_translator_fix_header(self): 

452 """Check that translator classes can fix headers.""" 

453 

454 # Read in a known header 

455 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

456 self.assertEqual(header["DTSITE"], "ct") 

457 

458 header2 = copy.copy(header) 

459 fixed = fix_header(header2, translator_class=NotDecamTranslator) 

460 self.assertTrue(fixed) 

461 self.assertEqual(header2["DTSITE"], "hi") 

462 

463 header2 = copy.copy(header) 

464 header2["DTSITE"] = "reset" 

465 with self.assertLogs("astro_metadata_translator", level="FATAL"): 

466 fixed = fix_header(header2, translator_class=AlsoNotDecamTranslator) 

467 self.assertFalse(fixed) 

468 self.assertEqual(header2["DTSITE"], "reset") 

469 

470 def test_no_double_fix(self): 

471 """Check that header fixup only happens once.""" 

472 

473 # Read in a known header 

474 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

475 self.assertEqual(header["DTSITE"], "ct") 

476 

477 # First time it will modifiy DTSITE 

478 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

479 self.assertTrue(fixed) 

480 self.assertEqual(header["DTSITE"], "cthi") 

481 

482 # Get the fix up date 

483 date = header["HIERARCH ASTRO METADATA FIX DATE"] 

484 

485 # Second time it will do nothing but still report it was fixed 

486 fixed = fix_header(header, translator_class=NotDecamTranslator2) 

487 self.assertTrue(fixed) 

488 self.assertEqual(header["DTSITE"], "cthi") 

489 

490 # Date of fixup should be the same 

491 self.assertEqual(header["HIERARCH ASTRO METADATA FIX DATE"], date) 

492 

493 # Test the translator version in provenance 

494 self.assertEqual(header["HIERARCH ASTRO METADATA FIX VERSION"], "1.0.0") 

495 

496 

497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 unittest.main()