Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import unittest 

13import os.path 

14 

15from astro_metadata_translator import merge_headers, fix_header, HscTranslator 

16from astro_metadata_translator.tests import read_test_file 

17from astro_metadata_translator import DecamTranslator 

18 

19TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

20 

21 

22class NotDecamTranslator(DecamTranslator): 

23 """This is a DECam translator with override list of header corrections.""" 

24 name = None 

25 

26 @classmethod 

27 def fix_header(cls, header, instrument, obsid, filename=None): 

28 header["DTSITE"] = "hi" 

29 return True 

30 

31 

32class AlsoNotDecamTranslator(DecamTranslator): 

33 """This is a DECam translator with override list of header corrections 

34 that fails.""" 

35 name = None 

36 

37 @classmethod 

38 def fix_header(cls, header, instrument, obsid, filename=None): 

39 raise RuntimeError("Failure to work something out from header") 

40 

41 

42class NullDecamTranslator(DecamTranslator): 

43 """This is a DECam translator that doesn't do any fixes.""" 

44 name = None 

45 

46 @classmethod 

47 def fix_header(cls, header, instrument, obsid, filename=None): 

48 return False 

49 

50 

51class HeadersTestCase(unittest.TestCase): 

52 

53 def setUp(self): 

54 # Define reference headers 

55 self.h1 = dict( 

56 ORIGIN="LSST", 

57 KEY0=0, 

58 KEY1=1, 

59 KEY2=3, 

60 KEY3=3.1415, 

61 KEY4="a", 

62 ) 

63 

64 self.h2 = dict( 

65 ORIGIN="LSST", 

66 KEY0="0", 

67 KEY2=4, 

68 KEY5=42 

69 ) 

70 self.h3 = dict( 

71 ORIGIN="AUXTEL", 

72 KEY3=3.1415, 

73 KEY2=50, 

74 KEY5=42, 

75 ) 

76 self.h4 = dict( 

77 KEY6="New", 

78 KEY1="Exists", 

79 ) 

80 

81 # Add keys for sorting by time 

82 # Sorted order: h2, h1, h4, h3 

83 self.h1["MJD-OBS"] = 50000.0 

84 self.h2["MJD-OBS"] = 49000.0 

85 self.h3["MJD-OBS"] = 53000.0 

86 self.h4["MJD-OBS"] = 52000.0 

87 

88 def test_fail(self): 

89 with self.assertRaises(ValueError): 

90 merge_headers([self.h1, self.h2], mode="wrong") 

91 

92 with self.assertRaises(ValueError): 

93 merge_headers([]) 

94 

95 def test_one(self): 

96 merged = merge_headers([self.h1], mode="drop") 

97 self.assertEqual(merged, self.h1) 

98 

99 def test_merging_overwrite(self): 

100 merged = merge_headers([self.h1, self.h2], mode="overwrite") 

101 # The merged header should be the same type as the first header 

102 self.assertIsInstance(merged, type(self.h1)) 

103 

104 expected = { 

105 "MJD-OBS": self.h2["MJD-OBS"], 

106 "ORIGIN": self.h2["ORIGIN"], 

107 "KEY0": self.h2["KEY0"], 

108 "KEY1": self.h1["KEY1"], 

109 "KEY2": self.h2["KEY2"], 

110 "KEY3": self.h1["KEY3"], 

111 "KEY4": self.h1["KEY4"], 

112 "KEY5": self.h2["KEY5"], 

113 } 

114 self.assertEqual(merged, expected) 

115 

116 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

117 mode="overwrite") 

118 

119 expected = { 

120 "MJD-OBS": self.h4["MJD-OBS"], 

121 "ORIGIN": self.h3["ORIGIN"], 

122 "KEY0": self.h2["KEY0"], 

123 "KEY1": self.h4["KEY1"], 

124 "KEY2": self.h3["KEY2"], 

125 "KEY3": self.h3["KEY3"], 

126 "KEY4": self.h1["KEY4"], 

127 "KEY5": self.h3["KEY5"], 

128 "KEY6": self.h4["KEY6"], 

129 } 

130 

131 self.assertEqual(merged, expected) 

132 

133 def test_merging_first(self): 

134 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

135 mode="first") 

136 

137 expected = { 

138 "MJD-OBS": self.h1["MJD-OBS"], 

139 "ORIGIN": self.h1["ORIGIN"], 

140 "KEY0": self.h1["KEY0"], 

141 "KEY1": self.h1["KEY1"], 

142 "KEY2": self.h1["KEY2"], 

143 "KEY3": self.h1["KEY3"], 

144 "KEY4": self.h1["KEY4"], 

145 "KEY5": self.h2["KEY5"], 

146 "KEY6": self.h4["KEY6"], 

147 } 

148 

149 self.assertEqual(merged, expected) 

150 

151 def test_merging_drop(self): 

152 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

153 mode="drop") 

154 

155 expected = { 

156 "KEY3": self.h1["KEY3"], 

157 "KEY4": self.h1["KEY4"], 

158 "KEY5": self.h2["KEY5"], 

159 "KEY6": self.h4["KEY6"], 

160 } 

161 

162 self.assertEqual(merged, expected) 

163 

164 # Sorting the headers should make no difference to drop mode 

165 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

166 mode="drop", sort=True) 

167 self.assertEqual(merged, expected) 

168 

169 # Now retain some headers 

170 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

171 mode="drop", sort=False, first=["ORIGIN"], last=["KEY2", "KEY1"]) 

172 

173 expected = { 

174 "KEY2": self.h3["KEY2"], 

175 "ORIGIN": self.h1["ORIGIN"], 

176 "KEY1": self.h4["KEY1"], 

177 "KEY3": self.h1["KEY3"], 

178 "KEY4": self.h1["KEY4"], 

179 "KEY5": self.h2["KEY5"], 

180 "KEY6": self.h4["KEY6"], 

181 } 

182 self.assertEqual(merged, expected) 

183 

184 # Now retain some headers with sorting 

185 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

186 mode="drop", sort=True, first=["ORIGIN"], last=["KEY2", "KEY1"]) 

187 

188 expected = { 

189 "KEY2": self.h3["KEY2"], 

190 "ORIGIN": self.h2["ORIGIN"], 

191 "KEY1": self.h4["KEY1"], 

192 "KEY3": self.h1["KEY3"], 

193 "KEY4": self.h1["KEY4"], 

194 "KEY5": self.h2["KEY5"], 

195 "KEY6": self.h4["KEY6"], 

196 } 

197 self.assertEqual(merged, expected) 

198 

199 def test_merging_diff(self): 

200 self.maxDiff = None 

201 

202 # Nothing in common for diff 

203 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

204 mode="diff") 

205 

206 expected = { 

207 "__DIFF__": [self.h1, self.h2, self.h3, self.h4] 

208 } 

209 

210 self.assertEqual(merged, expected) 

211 

212 # Now with a subset that does have overlap 

213 merged = merge_headers([self.h1, self.h2], 

214 mode="diff") 

215 expected = { 

216 "ORIGIN": "LSST", 

217 "__DIFF__": [ 

218 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

219 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

220 ] 

221 } 

222 self.assertEqual(merged, expected) 

223 

224 # Reverse to make sure there is nothing special about the first header 

225 merged = merge_headers([self.h2, self.h1], 

226 mode="diff") 

227 expected = { 

228 "ORIGIN": "LSST", 

229 "__DIFF__": [ 

230 {k: self.h2[k] for k in ("KEY0", "KEY2", "KEY5", "MJD-OBS")}, 

231 {k: self.h1[k] for k in ("KEY0", "KEY1", "KEY2", "KEY3", "KEY4", "MJD-OBS")}, 

232 ] 

233 } 

234 self.assertEqual(merged, expected) 

235 

236 # Check that identical headers have empty diff 

237 merged = merge_headers([self.h1, self.h1], 

238 mode="diff") 

239 expected = { 

240 **self.h1, 

241 "__DIFF__": [ 

242 {}, 

243 {}, 

244 ] 

245 } 

246 self.assertEqual(merged, expected) 

247 

248 def test_merging_append(self): 

249 # Try with two headers first 

250 merged = merge_headers([self.h1, self.h2], mode="append") 

251 

252 expected = { 

253 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"]], 

254 "ORIGIN": self.h1["ORIGIN"], 

255 "KEY0": [self.h1["KEY0"], self.h2["KEY0"]], 

256 "KEY1": self.h1["KEY1"], 

257 "KEY2": [self.h1["KEY2"], self.h2["KEY2"]], 

258 "KEY3": self.h1["KEY3"], 

259 "KEY4": self.h1["KEY4"], 

260 "KEY5": self.h2["KEY5"], 

261 } 

262 

263 self.assertEqual(merged, expected) 

264 

265 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

266 mode="append") 

267 

268 expected = { 

269 "MJD-OBS": [self.h1["MJD-OBS"], self.h2["MJD-OBS"], self.h3["MJD-OBS"], self.h4["MJD-OBS"]], 

270 "ORIGIN": [self.h1["ORIGIN"], self.h2["ORIGIN"], self.h3["ORIGIN"], None], 

271 "KEY0": [self.h1["KEY0"], self.h2["KEY0"], None, None], 

272 "KEY1": [self.h1["KEY1"], None, None, self.h4["KEY1"]], 

273 "KEY2": [self.h1["KEY2"], self.h2["KEY2"], self.h3["KEY2"], None], 

274 "KEY3": self.h3["KEY3"], 

275 "KEY4": self.h1["KEY4"], 

276 "KEY5": self.h3["KEY5"], 

277 "KEY6": self.h4["KEY6"], 

278 } 

279 

280 self.assertEqual(merged, expected) 

281 

282 def test_merging_overwrite_sort(self): 

283 merged = merge_headers([self.h1, self.h2], mode="overwrite", sort=True) 

284 

285 expected = { 

286 "MJD-OBS": self.h1["MJD-OBS"], 

287 "ORIGIN": self.h1["ORIGIN"], 

288 "KEY0": self.h1["KEY0"], 

289 "KEY1": self.h1["KEY1"], 

290 "KEY2": self.h1["KEY2"], 

291 "KEY3": self.h1["KEY3"], 

292 "KEY4": self.h1["KEY4"], 

293 "KEY5": self.h2["KEY5"], 

294 } 

295 self.assertEqual(merged, expected) 

296 

297 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

298 mode="overwrite", sort=True) 

299 

300 expected = { 

301 "MJD-OBS": self.h3["MJD-OBS"], 

302 "ORIGIN": self.h3["ORIGIN"], 

303 "KEY0": self.h1["KEY0"], 

304 "KEY1": self.h4["KEY1"], 

305 "KEY2": self.h3["KEY2"], 

306 "KEY3": self.h3["KEY3"], 

307 "KEY4": self.h1["KEY4"], 

308 "KEY5": self.h3["KEY5"], 

309 "KEY6": self.h4["KEY6"], 

310 } 

311 

312 self.assertEqual(merged, expected) 

313 

314 # Changing the order should not change the result 

315 merged = merge_headers([self.h4, self.h1, self.h3, self.h2], 

316 mode="overwrite", sort=True) 

317 

318 self.assertEqual(merged, expected) 

319 

320 def test_merging_first_sort(self): 

321 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

322 mode="first", sort=True) 

323 

324 expected = { 

325 "MJD-OBS": self.h2["MJD-OBS"], 

326 "ORIGIN": self.h2["ORIGIN"], 

327 "KEY0": self.h2["KEY0"], 

328 "KEY1": self.h1["KEY1"], 

329 "KEY2": self.h2["KEY2"], 

330 "KEY3": self.h1["KEY3"], 

331 "KEY4": self.h1["KEY4"], 

332 "KEY5": self.h2["KEY5"], 

333 "KEY6": self.h4["KEY6"], 

334 } 

335 

336 self.assertEqual(merged, expected) 

337 

338 def test_merging_append_sort(self): 

339 # Try with two headers first 

340 merged = merge_headers([self.h1, self.h2], mode="append", sort=True) 

341 

342 expected = { 

343 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"]], 

344 "ORIGIN": self.h1["ORIGIN"], 

345 "KEY0": [self.h2["KEY0"], self.h1["KEY0"]], 

346 "KEY1": self.h1["KEY1"], 

347 "KEY2": [self.h2["KEY2"], self.h1["KEY2"]], 

348 "KEY3": self.h1["KEY3"], 

349 "KEY4": self.h1["KEY4"], 

350 "KEY5": self.h2["KEY5"], 

351 } 

352 

353 self.assertEqual(merged, expected) 

354 

355 merged = merge_headers([self.h1, self.h2, self.h3, self.h4], 

356 mode="append", sort=True) 

357 

358 expected = { 

359 "MJD-OBS": [self.h2["MJD-OBS"], self.h1["MJD-OBS"], self.h4["MJD-OBS"], self.h3["MJD-OBS"]], 

360 "ORIGIN": [self.h2["ORIGIN"], self.h1["ORIGIN"], None, self.h3["ORIGIN"]], 

361 "KEY0": [self.h2["KEY0"], self.h1["KEY0"], None, None], 

362 "KEY1": [None, self.h1["KEY1"], self.h4["KEY1"], None], 

363 "KEY2": [self.h2["KEY2"], self.h1["KEY2"], None, self.h3["KEY2"]], 

364 "KEY3": self.h3["KEY3"], 

365 "KEY4": self.h1["KEY4"], 

366 "KEY5": self.h3["KEY5"], 

367 "KEY6": self.h4["KEY6"], 

368 } 

369 

370 self.assertEqual(merged, expected) 

371 

372 # Order should not matter 

373 merged = merge_headers([self.h4, self.h3, self.h2, self.h1], 

374 mode="append", sort=True) 

375 self.assertEqual(merged, expected) 

376 

377 

378class FixHeadersTestCase(unittest.TestCase): 

379 

380 def test_basic_fix_header(self): 

381 """Test that a header can be fixed if we specify a local path. 

382 """ 

383 

384 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

385 self.assertEqual(header["DETECTOR"], "S3-111_107419-8-3") 

386 

387 # First fix header but using no search path (should work as no-op) 

388 fixed = fix_header(header, translator_class=NullDecamTranslator) 

389 self.assertFalse(fixed) 

390 

391 # Now using the test corrections directory 

392 fixed = fix_header(header, search_path=os.path.join(TESTDIR, "data", "corrections"), 

393 translator_class=NullDecamTranslator) 

394 self.assertTrue(fixed) 

395 self.assertEqual(header["DETECTOR"], "NEW-ID") 

396 

397 # Now with a corrections directory that has bad YAML in it 

398 with self.assertLogs(level="WARN"): 

399 fixed = fix_header(header, search_path=os.path.join(TESTDIR, "data", "bad_corrections"), 

400 translator_class=NullDecamTranslator) 

401 self.assertFalse(fixed) 

402 

403 # Test that fix_header of unknown header is allowed 

404 header = {"SOMETHING": "UNKNOWN"} 

405 fixed = fix_header(header, translator_class=NullDecamTranslator) 

406 self.assertFalse(fixed) 

407 

408 def test_hsc_fix_header(self): 

409 """Check that one of the known HSC corrections is being applied 

410 properly.""" 

411 header = {"EXP-ID": "HSCA00120800", 

412 "INSTRUME": "HSC", 

413 "DATA-TYP": "FLAT"} 

414 

415 fixed = fix_header(header, translator_class=HscTranslator) 

416 self.assertTrue(fixed) 

417 self.assertEqual(header["DATA-TYP"], "OBJECT") 

418 

419 # And that this header won't be corrected 

420 header = {"EXP-ID": "HSCA00120800X", 

421 "INSTRUME": "HSC", 

422 "DATA-TYP": "FLAT"} 

423 

424 fixed = fix_header(header, translator_class=HscTranslator) 

425 self.assertFalse(fixed) 

426 self.assertEqual(header["DATA-TYP"], "FLAT") 

427 

428 def test_decam_fix_header(self): 

429 """Check that one of the known DECam corrections is being applied 

430 properly.""" 

431 

432 # This header is a bias (zero) with an erroneous Y filter 

433 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

434 fixed = fix_header(header, translator_class=DecamTranslator) 

435 self.assertTrue(fixed) 

436 self.assertEqual(header["FILTER"], "solid plate 0.0 0.0") 

437 

438 def test_translator_fix_header(self): 

439 """Check that translator classes can fix headers.""" 

440 

441 # Read in a known header 

442 header = read_test_file("fitsheader-decam-0160496.yaml", dir=os.path.join(TESTDIR, "data")) 

443 self.assertEqual(header["DTSITE"], "ct") 

444 fixed = fix_header(header, translator_class=NotDecamTranslator) 

445 self.assertTrue(fixed) 

446 self.assertEqual(header["DTSITE"], "hi") 

447 

448 header["DTSITE"] = "reset" 

449 with self.assertLogs("astro_metadata_translator", level="FATAL"): 

450 fixed = fix_header(header, translator_class=AlsoNotDecamTranslator) 

451 self.assertFalse(fixed) 

452 self.assertEqual(header["DTSITE"], "reset") 

453 

454 

455if __name__ == "__main__": 455 ↛ 456line 455 didn't jump to line 456, because the condition on line 455 was never true

456 unittest.main()