Coverage for python/astro_metadata_translator/bin/translateheader.py: 9%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

114 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Implementation of the ``translate_header.py`` script. 

13 

14Read file metadata from the specified files and report the translated content. 

15""" 

16 

17__all__ = ("main", "process_files") 

18 

19import argparse 

20import logging 

21 

22import sys 

23import traceback 

24import importlib 

25import yaml 

26from astro_metadata_translator import ObservationInfo, fix_header, MetadataTranslator 

27 

28from ..file_helpers import find_files, read_basic_metadata_from_file 

29 

30 

31# Output mode choices 

32OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none") 

33 

34# Definitions for table columns 

35TABLE_COLUMNS = ({ 

36 "format": "32.32s", 

37 "attr": "observation_id", 

38 "label": "ObsId" 

39 }, 

40 { 

41 "format": "8.8s", 

42 "attr": "observation_type", 

43 "label": "ImgType", 

44 }, 

45 { 

46 "format": "16.16s", 

47 "attr": "object", 

48 "label": "Object", 

49 }, 

50 { 

51 "format": "16.16s", 

52 "attr": "physical_filter", 

53 "label": "Filter", 

54 }, 

55 { 

56 "format": ">8.8s", 

57 "attr": "detector_unique_name", 

58 "label": "Detector" 

59 }, 

60 { 

61 "format": "5.1f", 

62 "attr": "exposure_time", 

63 "label": "ExpTime", 

64 }, 

65 ) 

66 

67 

68def build_argparser(): 

69 """Construct an argument parser for the ``translate_header.py`` script. 

70 

71 Returns 

72 ------- 

73 argparser : `argparse.ArgumentParser` 

74 The argument parser that defines the ``translate_header.py`` 

75 command-line interface. 

76 """ 

77 

78 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files") 

79 parser.add_argument("files", metavar="file", type=str, nargs="+", 

80 help="File(s) from which headers will be parsed." 

81 " If a directory is given it will be scanned for files matching the regular" 

82 " expression defined in --regex.") 

83 parser.add_argument("-q", "--quiet", action="store_true", 

84 help="Do not report the translation content from each header. This forces " 

85 "output mode 'none'.") 

86 parser.add_argument("-d", "--dumphdr", action="store_true", 

87 help="Dump the header in YAML format to standard output rather than translating it." 

88 " This is the same as using mode=yaml") 

89 parser.add_argument("--traceback", action="store_true", 

90 help="Give detailed trace back when any errors encountered") 

91 parser.add_argument("-n", "--hdrnum", default=1, 

92 help="HDU number to read. If the HDU can not be found, a warning is issued but " 

93 "translation is attempted using the primary header. " 

94 "The primary header is always read and merged with this header.") 

95 parser.add_argument("-m", "--mode", default="auto", choices=OUTPUT_MODES, 

96 help="Display mode for translated parameters. 'verbose' displays all the information" 

97 " available. 'table' displays important information in tabular form." 

98 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)." 

99 " 'fixed' dumps the header in YAML after it has had corrections applied." 

100 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form." 

101 " 'none' displays no translated header information and is an alias for the " 

102 " '--quiet' option." 

103 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.") 

104 parser.add_argument("-l", "--log", default="warn", 

105 help="Python logging level to use.") 

106 

107 re_default = r"\.fit[s]?\b" 

108 parser.add_argument("-r", "--regex", default=re_default, 

109 help="When looking in a directory, regular expression to use to determine whether" 

110 f" a file should be examined. Default: '{re_default}'") 

111 

112 parser.add_argument("-p", "--packages", action="append", type=str, 

113 help="Python packages to import to register additional translators") 

114 

115 return parser 

116 

117 

118def read_file(file, hdrnum, print_trace, 

119 outstream=sys.stdout, errstream=sys.stderr, output_mode="verbose", 

120 write_heading=False): 

121 """Read the specified file and process it. 

122 

123 Parameters 

124 ---------- 

125 file : `str` 

126 The file from which the header is to be read. 

127 hdrnum : `int` 

128 The HDU number to read. The primary header is always read and 

129 merged with the header from this HDU. 

130 print_trace : `bool` 

131 If there is an error reading the file and this parameter is `True`, 

132 a full traceback of the exception will be reported. If `False` prints 

133 a one line summary of the error condition. 

134 outstream : `io.StringIO`, optional 

135 Output stream to use for standard messages. Defaults to `sys.stdout`. 

136 errstream : `io.StringIO`, optional 

137 Stream to send messages that would normally be sent to standard 

138 error. Defaults to `sys.stderr`. 

139 output_mode : `str`, optional 

140 Output mode to use. Must be one of "verbose", "none", "table", 

141 "yaml", or "fixed". "yaml" and "fixed" can be modified with a 

142 "native" suffix to indicate that the output should be a representation 

143 of the native object type representing the header (which can be 

144 PropertyList or an Astropy header). Without this modify headers 

145 will be dumped as simple `dict` form. 

146 "auto" is used to indicate that a single file has been specified 

147 but the output will depend on whether the file is a multi-extension 

148 FITS file or not. 

149 write_heading: `bool`, optional 

150 If `True` and in table mode, write a table heading out before writing 

151 the content. 

152 

153 Returns 

154 ------- 

155 success : `bool` 

156 `True` if the file was handled successfully, `False` if the file 

157 could not be processed. 

158 """ 

159 if output_mode not in OUTPUT_MODES: 

160 raise ValueError(f"Output mode of '{output_mode}' is not understood.") 

161 

162 # This gets in the way in tabular mode 

163 if output_mode != "table": 

164 print(f"Analyzing {file}...", file=errstream) 

165 

166 try: 

167 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, can_raise=True) 

168 if md is None: 

169 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}") 

170 

171 if output_mode.endswith("native"): 

172 # Strip native and don't change type of md 

173 output_mode = output_mode[:-len("native")] 

174 else: 

175 # Rewrite md as simple dict for output 

176 md = {k: v for k, v in md.items()} 

177 

178 if output_mode in ("yaml", "fixed"): 

179 

180 if output_mode == "fixed": 

181 fix_header(md, filename=file) 

182 

183 # The header should be written out in the insertion order 

184 print(yaml.dump(md, sort_keys=False), file=outstream) 

185 return True 

186 

187 # Try to work out a translator class. 

188 translator_class = MetadataTranslator.determine_translator(md, filename=file) 

189 

190 # Work out which headers to translate, assuming the default if 

191 # we have a YAML test file. 

192 if file.endswith(".yaml"): 

193 headers = [md] 

194 else: 

195 headers = list(translator_class.determine_translatable_headers(file, md)) 

196 if output_mode == "auto": 

197 output_mode = "table" if len(headers) > 1 else "verbose" 

198 

199 wrote_heading = False 

200 for md in headers: 

201 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

202 if output_mode == "table": 

203 columns = ["{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) 

204 for c in TABLE_COLUMNS] 

205 

206 if write_heading and not wrote_heading: 

207 # Construct headings of the same width as the items 

208 # we have calculated. Doing this means we don't have to 

209 # work out for ourselves how many characters will be used 

210 # for non-strings (especially Quantity) 

211 headings = [] 

212 separators = [] 

213 for thiscol, defn in zip(columns, TABLE_COLUMNS): 

214 width = len(thiscol) 

215 headings.append("{:{w}.{w}}".format(defn["label"], w=width)) 

216 separators.append("-"*width) 

217 print(" ".join(headings), file=outstream) 

218 print(" ".join(separators), file=outstream) 

219 wrote_heading = True 

220 

221 row = " ".join(columns) 

222 print(row, file=outstream) 

223 elif output_mode == "verbose": 

224 print(f"{obs_info}", file=outstream) 

225 elif output_mode == "none": 

226 pass 

227 else: 

228 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.") 

229 except Exception as e: 

230 if print_trace: 

231 traceback.print_exc(file=outstream) 

232 else: 

233 print(f"Failure processing {file}: {e}", file=outstream) 

234 return False 

235 return True 

236 

237 

238def process_files(files, regex, hdrnum, print_trace, 

239 outstream=sys.stdout, errstream=sys.stderr, 

240 output_mode="auto"): 

241 """Read and translate metadata from the specified files. 

242 

243 Parameters 

244 ---------- 

245 files : iterable of `str` 

246 The files or directories from which the headers are to be read. 

247 regex : `str` 

248 Regular expression string used to filter files when a directory is 

249 scanned. 

250 hdrnum : `int` 

251 The HDU number to read. The primary header is always read and 

252 merged with the header from this HDU. 

253 print_trace : `bool` 

254 If there is an error reading the file and this parameter is `True`, 

255 a full traceback of the exception will be reported. If `False` prints 

256 a one line summary of the error condition. 

257 outstream : `io.StringIO`, optional 

258 Output stream to use for standard messages. Defaults to `sys.stdout`. 

259 errstream : `io.StringIO`, optional 

260 Stream to send messages that would normally be sent to standard 

261 error. Defaults to `sys.stderr`. 

262 output_mode : `str`, optional 

263 Output mode to use for the translated information. 

264 "auto" switches based on how many files are found. 

265 

266 Returns 

267 ------- 

268 okay : `list` of `str` 

269 All the files that were processed successfully. 

270 failed : `list` of `str` 

271 All the files that could not be processed. 

272 """ 

273 found_files = find_files(files, regex) 

274 

275 # Convert "auto" to correct mode but for a single file keep it 

276 # auto in case that file has multiple headers 

277 if output_mode == "auto": 

278 if len(found_files) > 1: 

279 output_mode = "table" 

280 

281 # Process each file 

282 failed = [] 

283 okay = [] 

284 heading = True 

285 for path in sorted(found_files): 

286 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode, 

287 heading) 

288 heading = False 

289 if isok: 

290 okay.append(path) 

291 else: 

292 failed.append(path) 

293 

294 return okay, failed 

295 

296 

297def main(): 

298 """Read metadata from the supplied files and translate the content to 

299 standard form. 

300 

301 Returns 

302 ------- 

303 status : `int` 

304 Exit status to be passed to `sys.exit()`. 0 if any of the files 

305 could be translated. 1 otherwise. 

306 """ 

307 

308 logging.warn("This command is deprecated. Please use 'astrometadata translate' " 

309 " or 'astrometadata dump' instead. See 'astrometadata -h' for more details.") 

310 

311 args = build_argparser().parse_args() 

312 

313 # Process import requests 

314 if args.packages: 

315 for m in args.packages: 

316 importlib.import_module(m) 

317 

318 output_mode = args.mode 

319 if args.quiet: 

320 output_mode = "none" 

321 elif args.dumphdr: 

322 output_mode = "yaml" 

323 

324 # Set the log level. Convert to upper case to allow the user to 

325 # specify --log=DEBUG or --log=debug 

326 numeric_level = getattr(logging, args.log.upper(), None) 

327 if not isinstance(numeric_level, int): 

328 raise ValueError(f"Invalid log level: {args.log}") 

329 logging.basicConfig(level=numeric_level) 

330 

331 # Main loop over files 

332 okay, failed = process_files(args.files, args.regex, args.hdrnum, 

333 args.traceback, 

334 output_mode=output_mode) 

335 

336 if failed: 

337 print("Files with failed translations:", file=sys.stderr) 

338 for f in failed: 

339 print(f"\t{f}", file=sys.stderr) 

340 

341 if okay: 

342 # Good status if anything was returned in okay 

343 return 0 

344 else: 

345 return 1