Coverage for python/astro_metadata_translator/bin/translateheader.py: 10%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

116 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Implementation of the ``translate_header.py`` script. 

13 

14Read file metadata from the specified files and report the translated content. 

15""" 

16 

17from __future__ import annotations 

18 

19__all__ = ("main", "process_files") 

20 

21import argparse 

22import importlib 

23import logging 

24import sys 

25import traceback 

26from typing import IO, List, Sequence, Tuple 

27 

28import yaml 

29 

30from astro_metadata_translator import MetadataTranslator, ObservationInfo, fix_header 

31 

32from ..file_helpers import find_files, read_basic_metadata_from_file 

33 

34# Output mode choices 

35OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none") 

36 

37# Definitions for table columns 

38TABLE_COLUMNS = ( 

39 {"format": "32.32s", "attr": "observation_id", "label": "ObsId"}, 

40 { 

41 "format": "8.8s", 

42 "attr": "observation_type", 

43 "label": "ImgType", 

44 }, 

45 { 

46 "format": "16.16s", 

47 "attr": "object", 

48 "label": "Object", 

49 }, 

50 { 

51 "format": "16.16s", 

52 "attr": "physical_filter", 

53 "label": "Filter", 

54 }, 

55 {"format": ">8.8s", "attr": "detector_unique_name", "label": "Detector"}, 

56 { 

57 "format": "5.1f", 

58 "attr": "exposure_time", 

59 "label": "ExpTime", 

60 }, 

61) 

62 

63 

64def build_argparser() -> argparse.ArgumentParser: 

65 """Construct an argument parser for the ``translate_header.py`` script. 

66 

67 Returns 

68 ------- 

69 argparser : `argparse.ArgumentParser` 

70 The argument parser that defines the ``translate_header.py`` 

71 command-line interface. 

72 """ 

73 

74 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files") 

75 parser.add_argument( 

76 "files", 

77 metavar="file", 

78 type=str, 

79 nargs="+", 

80 help="File(s) from which headers will be parsed." 

81 " If a directory is given it will be scanned for files matching the regular" 

82 " expression defined in --regex.", 

83 ) 

84 parser.add_argument( 

85 "-q", 

86 "--quiet", 

87 action="store_true", 

88 help="Do not report the translation content from each header. This forces output mode 'none'.", 

89 ) 

90 parser.add_argument( 

91 "-d", 

92 "--dumphdr", 

93 action="store_true", 

94 help="Dump the header in YAML format to standard output rather than translating it." 

95 " This is the same as using mode=yaml", 

96 ) 

97 parser.add_argument( 

98 "--traceback", action="store_true", help="Give detailed trace back when any errors encountered" 

99 ) 

100 parser.add_argument( 

101 "-n", 

102 "--hdrnum", 

103 default=1, 

104 help="HDU number to read. If the HDU can not be found, a warning is issued but " 

105 "translation is attempted using the primary header. " 

106 "The primary header is always read and merged with this header.", 

107 ) 

108 parser.add_argument( 

109 "-m", 

110 "--mode", 

111 default="auto", 

112 choices=OUTPUT_MODES, 

113 help="Display mode for translated parameters. 'verbose' displays all the information" 

114 " available. 'table' displays important information in tabular form." 

115 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)." 

116 " 'fixed' dumps the header in YAML after it has had corrections applied." 

117 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form." 

118 " 'none' displays no translated header information and is an alias for the " 

119 " '--quiet' option." 

120 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.", 

121 ) 

122 parser.add_argument("-l", "--log", default="warn", help="Python logging level to use.") 

123 

124 re_default = r"\.fit[s]?\b" 

125 parser.add_argument( 

126 "-r", 

127 "--regex", 

128 default=re_default, 

129 help="When looking in a directory, regular expression to use to determine whether" 

130 f" a file should be examined. Default: '{re_default}'", 

131 ) 

132 

133 parser.add_argument( 

134 "-p", 

135 "--packages", 

136 action="append", 

137 type=str, 

138 help="Python packages to import to register additional translators", 

139 ) 

140 

141 return parser 

142 

143 

144def read_file( 

145 file: str, 

146 hdrnum: int, 

147 print_trace: bool, 

148 outstream: IO = sys.stdout, 

149 errstream: IO = sys.stderr, 

150 output_mode: str = "verbose", 

151 write_heading: bool = False, 

152) -> bool: 

153 """Read the specified file and process it. 

154 

155 Parameters 

156 ---------- 

157 file : `str` 

158 The file from which the header is to be read. 

159 hdrnum : `int` 

160 The HDU number to read. The primary header is always read and 

161 merged with the header from this HDU. 

162 print_trace : `bool` 

163 If there is an error reading the file and this parameter is `True`, 

164 a full traceback of the exception will be reported. If `False` prints 

165 a one line summary of the error condition. 

166 outstream : `io.StringIO`, optional 

167 Output stream to use for standard messages. Defaults to `sys.stdout`. 

168 errstream : `io.StringIO`, optional 

169 Stream to send messages that would normally be sent to standard 

170 error. Defaults to `sys.stderr`. 

171 output_mode : `str`, optional 

172 Output mode to use. Must be one of "verbose", "none", "table", 

173 "yaml", or "fixed". "yaml" and "fixed" can be modified with a 

174 "native" suffix to indicate that the output should be a representation 

175 of the native object type representing the header (which can be 

176 PropertyList or an Astropy header). Without this modify headers 

177 will be dumped as simple `dict` form. 

178 "auto" is used to indicate that a single file has been specified 

179 but the output will depend on whether the file is a multi-extension 

180 FITS file or not. 

181 write_heading: `bool`, optional 

182 If `True` and in table mode, write a table heading out before writing 

183 the content. 

184 

185 Returns 

186 ------- 

187 success : `bool` 

188 `True` if the file was handled successfully, `False` if the file 

189 could not be processed. 

190 """ 

191 if output_mode not in OUTPUT_MODES: 

192 raise ValueError(f"Output mode of '{output_mode}' is not understood.") 

193 

194 # This gets in the way in tabular mode 

195 if output_mode != "table": 

196 print(f"Analyzing {file}...", file=errstream) 

197 

198 try: 

199 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, can_raise=True) 

200 if md is None: 

201 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}") 

202 

203 if output_mode.endswith("native"): 

204 # Strip native and don't change type of md 

205 output_mode = output_mode[: -len("native")] 

206 else: 

207 # Rewrite md as simple dict for output 

208 md = {k: v for k, v in md.items()} 

209 

210 if output_mode in ("yaml", "fixed"): 

211 

212 if output_mode == "fixed": 

213 fix_header(md, filename=file) 

214 

215 # The header should be written out in the insertion order 

216 print(yaml.dump(md, sort_keys=False), file=outstream) 

217 return True 

218 

219 # Try to work out a translator class. 

220 translator_class = MetadataTranslator.determine_translator(md, filename=file) 

221 

222 # Work out which headers to translate, assuming the default if 

223 # we have a YAML test file. 

224 if file.endswith(".yaml"): 

225 headers = [md] 

226 else: 

227 headers = list(translator_class.determine_translatable_headers(file, md)) 

228 if output_mode == "auto": 

229 output_mode = "table" if len(headers) > 1 else "verbose" 

230 

231 wrote_heading = False 

232 for md in headers: 

233 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

234 if output_mode == "table": 

235 columns = [ 

236 "{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) for c in TABLE_COLUMNS 

237 ] 

238 

239 if write_heading and not wrote_heading: 

240 # Construct headings of the same width as the items 

241 # we have calculated. Doing this means we don't have to 

242 # work out for ourselves how many characters will be used 

243 # for non-strings (especially Quantity) 

244 headings = [] 

245 separators = [] 

246 for thiscol, defn in zip(columns, TABLE_COLUMNS): 

247 width = len(thiscol) 

248 headings.append("{:{w}.{w}}".format(defn["label"], w=width)) 

249 separators.append("-" * width) 

250 print(" ".join(headings), file=outstream) 

251 print(" ".join(separators), file=outstream) 

252 wrote_heading = True 

253 

254 row = " ".join(columns) 

255 print(row, file=outstream) 

256 elif output_mode == "verbose": 

257 print(f"{obs_info}", file=outstream) 

258 elif output_mode == "none": 

259 pass 

260 else: 

261 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.") 

262 except Exception as e: 

263 if print_trace: 

264 traceback.print_exc(file=outstream) 

265 else: 

266 print(f"Failure processing {file}: {e}", file=outstream) 

267 return False 

268 return True 

269 

270 

271def process_files( 

272 files: Sequence[str], 

273 regex: str, 

274 hdrnum: int, 

275 print_trace: bool, 

276 outstream: IO = sys.stdout, 

277 errstream: IO = sys.stderr, 

278 output_mode: str = "auto", 

279) -> Tuple[List[str], List[str]]: 

280 """Read and translate metadata from the specified files. 

281 

282 Parameters 

283 ---------- 

284 files : iterable of `str` 

285 The files or directories from which the headers are to be read. 

286 regex : `str` 

287 Regular expression string used to filter files when a directory is 

288 scanned. 

289 hdrnum : `int` 

290 The HDU number to read. The primary header is always read and 

291 merged with the header from this HDU. 

292 print_trace : `bool` 

293 If there is an error reading the file and this parameter is `True`, 

294 a full traceback of the exception will be reported. If `False` prints 

295 a one line summary of the error condition. 

296 outstream : `io.StringIO`, optional 

297 Output stream to use for standard messages. Defaults to `sys.stdout`. 

298 errstream : `io.StringIO`, optional 

299 Stream to send messages that would normally be sent to standard 

300 error. Defaults to `sys.stderr`. 

301 output_mode : `str`, optional 

302 Output mode to use for the translated information. 

303 "auto" switches based on how many files are found. 

304 

305 Returns 

306 ------- 

307 okay : `list` of `str` 

308 All the files that were processed successfully. 

309 failed : `list` of `str` 

310 All the files that could not be processed. 

311 """ 

312 found_files = find_files(files, regex) 

313 

314 # Convert "auto" to correct mode but for a single file keep it 

315 # auto in case that file has multiple headers 

316 if output_mode == "auto": 

317 if len(found_files) > 1: 

318 output_mode = "table" 

319 

320 # Process each file 

321 failed = [] 

322 okay = [] 

323 heading = True 

324 for path in sorted(found_files): 

325 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode, heading) 

326 heading = False 

327 if isok: 

328 okay.append(path) 

329 else: 

330 failed.append(path) 

331 

332 return okay, failed 

333 

334 

335def main() -> int: 

336 """Read metadata from the supplied files and translate the content to 

337 standard form. 

338 

339 Returns 

340 ------- 

341 status : `int` 

342 Exit status to be passed to `sys.exit()`. 0 if any of the files 

343 could be translated. 1 otherwise. 

344 """ 

345 

346 logging.warn( 

347 "This command is deprecated. Please use 'astrometadata translate' " 

348 " or 'astrometadata dump' instead. See 'astrometadata -h' for more details." 

349 ) 

350 

351 args = build_argparser().parse_args() 

352 

353 # Process import requests 

354 if args.packages: 

355 for m in args.packages: 

356 importlib.import_module(m) 

357 

358 output_mode = args.mode 

359 if args.quiet: 

360 output_mode = "none" 

361 elif args.dumphdr: 

362 output_mode = "yaml" 

363 

364 # Set the log level. Convert to upper case to allow the user to 

365 # specify --log=DEBUG or --log=debug 

366 numeric_level = getattr(logging, args.log.upper(), None) 

367 if not isinstance(numeric_level, int): 

368 raise ValueError(f"Invalid log level: {args.log}") 

369 logging.basicConfig(level=numeric_level) 

370 

371 # Main loop over files 

372 okay, failed = process_files(args.files, args.regex, args.hdrnum, args.traceback, output_mode=output_mode) 

373 

374 if failed: 

375 print("Files with failed translations:", file=sys.stderr) 

376 for f in failed: 

377 print(f"\t{f}", file=sys.stderr) 

378 

379 if okay: 

380 # Good status if anything was returned in okay 

381 return 0 

382 else: 

383 return 1