Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Implementation of the ``translate_header.py`` script. 

13 

14Read file metadata from the specified files and report the translated content. 

15""" 

16 

17__all__ = ("main", "process_files") 

18 

19import argparse 

20import logging 

21 

22import os 

23import re 

24import sys 

25import traceback 

26import importlib 

27import yaml 

28from astro_metadata_translator import ObservationInfo, merge_headers, fix_header 

29from astro_metadata_translator.tests import read_test_file 

30 

31# Prefer afw over Astropy 

32try: 

33 from lsst.afw.fits import readMetadata 

34 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata 

35 

36 def read_metadata(file, hdu): 

37 try: 

38 return readMetadata(file, hdu=hdu) 

39 except lsst.afw.fits.FitsError: 

40 return None 

41 

42except ImportError: 

43 from astropy.io import fits 

44 

45 def read_metadata(file, hdu): 

46 fits_file = fits.open(file) 

47 try: 

48 header = fits_file[hdu].header 

49 except IndexError: 

50 header = None 

51 return header 

52 

53 

54# Output mode choices 

55OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none") 

56 

57# Definitions for table columns 

58TABLE_COLUMNS = ({ 

59 "format": "32.32s", 

60 "attr": "observation_id", 

61 "label": "ObsId" 

62 }, 

63 { 

64 "format": "8.8s", 

65 "attr": "observation_type", 

66 "label": "ImgType", 

67 }, 

68 { 

69 "format": "16.16s", 

70 "attr": "object", 

71 "label": "Object", 

72 }, 

73 { 

74 "format": "16.16s", 

75 "attr": "physical_filter", 

76 "label": "Filter", 

77 }, 

78 { 

79 "format": "5.1f", 

80 "attr": "exposure_time", 

81 "label": "ExpTime", 

82 }, 

83 ) 

84 

85 

86def build_argparser(): 

87 """Construct an argument parser for the ``translate_header.py`` script. 

88 

89 Returns 

90 ------- 

91 argparser : `argparse.ArgumentParser` 

92 The argument parser that defines the ``translate_header.py`` 

93 command-line interface. 

94 """ 

95 

96 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files") 

97 parser.add_argument("files", metavar="file", type=str, nargs="+", 

98 help="File(s) from which headers will be parsed." 

99 " If a directory is given it will be scanned for files matching the regular" 

100 " expression defined in --regex.") 

101 parser.add_argument("-q", "--quiet", action="store_true", 

102 help="Do not report the translation content from each header. This forces " 

103 "output mode 'none'.") 

104 parser.add_argument("-d", "--dumphdr", action="store_true", 

105 help="Dump the header in YAML format to standard output rather than translating it." 

106 " This is the same as using mode=yaml") 

107 parser.add_argument("--traceback", action="store_true", 

108 help="Give detailed trace back when any errors encountered") 

109 parser.add_argument("-n", "--hdrnum", default=1, 

110 help="HDU number to read. If the HDU can not be found, a warning is issued but " 

111 "translation is attempted using the primary header. " 

112 "The primary header is always read and merged with this header.") 

113 parser.add_argument("-m", "--mode", default="auto", choices=OUTPUT_MODES, 

114 help="Display mode for translated parameters. 'verbose' displays all the information" 

115 " available. 'table' displays important information in tabular form." 

116 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)." 

117 " 'fixed' dumps the header in YAML after it has had corrections applied." 

118 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form." 

119 " 'none' displays no translated header information and is an alias for the " 

120 " '--quiet' option." 

121 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.") 

122 parser.add_argument("-l", "--log", default="warn", 

123 help="Python logging level to use.") 

124 

125 re_default = r"\.fit[s]?\b" 

126 parser.add_argument("-r", "--regex", default=re_default, 

127 help="When looking in a directory, regular expression to use to determine whether" 

128 f" a file should be examined. Default: '{re_default}'") 

129 

130 parser.add_argument("-p", "--packages", action="append", type=str, 

131 help="Python packages to import to register additional translators") 

132 

133 return parser 

134 

135 

136def read_file(file, hdrnum, print_trace, 

137 outstream=sys.stdout, errstream=sys.stderr, output_mode="verbose", 

138 write_heading=False): 

139 """Read the specified file and process it. 

140 

141 Parameters 

142 ---------- 

143 file : `str` 

144 The file from which the header is to be read. 

145 hdrnum : `int` 

146 The HDU number to read. The primary header is always read and 

147 merged with the header from this HDU. 

148 print_trace : `bool` 

149 If there is an error reading the file and this parameter is `True`, 

150 a full traceback of the exception will be reported. If `False` prints 

151 a one line summary of the error condition. 

152 outstream : `io.StringIO`, optional 

153 Output stream to use for standard messages. Defaults to `sys.stdout`. 

154 errstream : `io.StringIO`, optional 

155 Stream to send messages that would normally be sent to standard 

156 error. Defaults to `sys.stderr`. 

157 output_mode : `str`, optional 

158 Output mode to use. Must be one of "verbose", "none", "table", 

159 "yaml", or "fixed". "yaml" and "fixed" can be modified with a 

160 "native" suffix to indicate that the output should be a representation 

161 of the native object type representing the header (which can be 

162 PropertyList or an Astropy header). Without this modify headers 

163 will be dumped as simple `dict` form. 

164 "auto" is not allowed by this point. 

165 write_heading: `bool`, optional 

166 If `True` and in table mode, write a table heading out before writing 

167 the content. 

168 

169 Returns 

170 ------- 

171 success : `bool` 

172 `True` if the file was handled successfully, `False` if the file 

173 could not be processed. 

174 """ 

175 if output_mode not in OUTPUT_MODES: 

176 raise ValueError(f"Output mode of '{output_mode}' is not understood.") 

177 if output_mode == "auto": 

178 raise ValueError("Output mode can not be 'auto' here.") 

179 

180 # This gets in the way in tabular mode 

181 if output_mode != "table": 

182 print(f"Analyzing {file}...", file=errstream) 

183 

184 try: 

185 if file.endswith(".yaml"): 

186 md = read_test_file(file,) 

187 if hdrnum != 0: 

188 # YAML can't have HDUs 

189 hdrnum = 0 

190 else: 

191 md = read_metadata(file, 0) 

192 if md is None: 

193 print(f"Unable to open file {file}", file=errstream) 

194 return False 

195 if hdrnum != 0: 

196 mdn = read_metadata(file, int(hdrnum)) 

197 # Astropy does not allow append mode since it does not 

198 # convert lists to multiple cards. Overwrite for now 

199 if mdn is not None: 

200 md = merge_headers([md, mdn], mode="overwrite") 

201 else: 

202 print(f"HDU {hdrnum} was not found. Ignoring request.", file=errstream) 

203 

204 if output_mode.endswith("native"): 

205 # Strip native and don't change type of md 

206 output_mode = output_mode[:-len("native")] 

207 else: 

208 # Rewrite md as simple dict for output 

209 md = {k: v for k, v in md.items()} 

210 

211 if output_mode in ("yaml", "fixed"): 

212 

213 if output_mode == "fixed": 

214 fix_header(md, filename=file) 

215 

216 # The header should be written out in the insertion order 

217 print(yaml.dump(md, sort_keys=False), file=outstream) 

218 return True 

219 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

220 if output_mode == "table": 

221 columns = ["{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) 

222 for c in TABLE_COLUMNS] 

223 

224 if write_heading: 

225 # Construct headings of the same width as the items 

226 # we have calculated. Doing this means we don't have to 

227 # work out for ourselves how many characters will be used 

228 # for non-strings (especially Quantity) 

229 headings = [] 

230 separators = [] 

231 for thiscol, defn in zip(columns, TABLE_COLUMNS): 

232 width = len(thiscol) 

233 headings.append("{:{w}.{w}}".format(defn["label"], w=width)) 

234 separators.append("-"*width) 

235 print(" ".join(headings), file=outstream) 

236 print(" ".join(separators), file=outstream) 

237 

238 row = " ".join(columns) 

239 print(row, file=outstream) 

240 elif output_mode == "verbose": 

241 print(f"{obs_info}", file=outstream) 

242 elif output_mode == "none": 

243 pass 

244 else: 

245 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.") 

246 except Exception as e: 

247 if print_trace: 

248 traceback.print_exc(file=outstream) 

249 else: 

250 print(repr(e), file=outstream) 

251 return False 

252 return True 

253 

254 

255def process_files(files, regex, hdrnum, print_trace, 

256 outstream=sys.stdout, errstream=sys.stderr, 

257 output_mode="auto"): 

258 """Read and translate metadata from the specified files. 

259 

260 Parameters 

261 ---------- 

262 files : iterable of `str` 

263 The files or directories from which the headers are to be read. 

264 regex : `str` 

265 Regular expression string used to filter files when a directory is 

266 scanned. 

267 hdrnum : `int` 

268 The HDU number to read. The primary header is always read and 

269 merged with the header from this HDU. 

270 print_trace : `bool` 

271 If there is an error reading the file and this parameter is `True`, 

272 a full traceback of the exception will be reported. If `False` prints 

273 a one line summary of the error condition. 

274 outstream : `io.StringIO`, optional 

275 Output stream to use for standard messages. Defaults to `sys.stdout`. 

276 errstream : `io.StringIO`, optional 

277 Stream to send messages that would normally be sent to standard 

278 error. Defaults to `sys.stderr`. 

279 output_mode : `str`, optional 

280 Output mode to use for the translated information. 

281 "auto" switches based on how many files are found. 

282 

283 Returns 

284 ------- 

285 okay : `list` of `str` 

286 All the files that were processed successfully. 

287 failed : `list` of `str` 

288 All the files that could not be processed. 

289 """ 

290 file_regex = re.compile(regex) 

291 found_files = [] 

292 

293 # Find all the files of interest 

294 for file in files: 

295 if os.path.isdir(file): 

296 for root, dirs, files in os.walk(file): 

297 for name in files: 

298 path = os.path.join(root, name) 

299 if os.path.isfile(path) and file_regex.search(name): 

300 found_files.append(path) 

301 else: 

302 found_files.append(file) 

303 

304 # Convert "auto" to correct mode 

305 if output_mode == "auto": 

306 if len(found_files) > 1: 

307 output_mode = "table" 

308 else: 

309 output_mode = "verbose" 

310 

311 # Process each file 

312 failed = [] 

313 okay = [] 

314 heading = True 

315 for path in sorted(found_files): 

316 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode, 

317 heading) 

318 heading = False 

319 if isok: 

320 okay.append(path) 

321 else: 

322 failed.append(path) 

323 

324 return okay, failed 

325 

326 

327def main(): 

328 """Read metadata from the supplied files and translate the content to 

329 standard form. 

330 

331 Returns 

332 ------- 

333 status : `int` 

334 Exit status to be passed to `sys.exit()`. 0 if any of the files 

335 could be translated. 1 otherwise. 

336 """ 

337 args = build_argparser().parse_args() 

338 

339 # Process import requests 

340 if args.packages: 

341 for m in args.packages: 

342 importlib.import_module(m) 

343 

344 output_mode = args.mode 

345 if args.quiet: 

346 output_mode = "none" 

347 elif args.dumphdr: 

348 output_mode = "yaml" 

349 

350 # Set the log level. Convert to upper case to allow the user to 

351 # specify --log=DEBUG or --log=debug 

352 numeric_level = getattr(logging, args.log.upper(), None) 

353 if not isinstance(numeric_level, int): 

354 raise ValueError(f"Invalid log level: {args.log}") 

355 logging.basicConfig(level=numeric_level) 

356 

357 # Main loop over files 

358 okay, failed = process_files(args.files, args.regex, args.hdrnum, 

359 args.traceback, 

360 output_mode=output_mode) 

361 

362 if failed: 

363 print("Files with failed translations:", file=sys.stderr) 

364 for f in failed: 

365 print(f"\t{f}", file=sys.stderr) 

366 

367 if okay: 

368 # Good status if anything was returned in okay 

369 return 0 

370 else: 

371 return 1