Coverage for python/astro_metadata_translator/bin/translate.py: 12%

78 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-20 03:54 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Implementation of the ``astrometadata translate`` command-line. 

13 

14Read file metadata from the specified files and report the translated content. 

15""" 

16 

17from __future__ import annotations 

18 

19__all__ = ("translate_or_dump_headers",) 

20 

21import sys 

22import traceback 

23from collections.abc import Sequence 

24from typing import IO 

25 

26import yaml 

27 

28from astro_metadata_translator import MetadataTranslator, ObservationInfo, fix_header 

29 

30from ..file_helpers import find_files, read_basic_metadata_from_file 

31 

32# Output mode choices 

33OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none") 

34 

35# Definitions for table columns 

36TABLE_COLUMNS = ( 

37 {"format": "32.32s", "attr": "observation_id", "label": "ObsId"}, 

38 { 

39 "format": "8.8s", 

40 "attr": "observation_type", 

41 "label": "ImgType", 

42 }, 

43 { 

44 "format": "16.16s", 

45 "attr": "object", 

46 "label": "Object", 

47 }, 

48 { 

49 "format": "16.16s", 

50 "attr": "physical_filter", 

51 "label": "Filter", 

52 }, 

53 {"format": ">8.8s", "attr": "detector_unique_name", "label": "Detector"}, 

54 { 

55 "format": "5.1f", 

56 "attr": "exposure_time", 

57 "label": "ExpTime", 

58 }, 

59) 

60 

61 

62def read_file( 

63 file: str, 

64 hdrnum: int, 

65 print_trace: bool, 

66 outstream: IO = sys.stdout, 

67 errstream: IO = sys.stderr, 

68 output_mode: str = "verbose", 

69 write_heading: bool = False, 

70) -> bool: 

71 """Read the specified file and process it. 

72 

73 Parameters 

74 ---------- 

75 file : `str` 

76 The file from which the header is to be read. 

77 hdrnum : `int` 

78 The HDU number to read. The primary header is always read and 

79 merged with the header from this HDU. 

80 print_trace : `bool` 

81 If there is an error reading the file and this parameter is `True`, 

82 a full traceback of the exception will be reported. If `False` prints 

83 a one line summary of the error condition. 

84 outstream : `io.StringIO`, optional 

85 Output stream to use for standard messages. Defaults to `sys.stdout`. 

86 errstream : `io.StringIO`, optional 

87 Stream to send messages that would normally be sent to standard 

88 error. Defaults to `sys.stderr`. 

89 output_mode : `str`, optional 

90 Output mode to use. Must be one of "verbose", "none", "table", 

91 "yaml", or "fixed". "yaml" and "fixed" can be modified with a 

92 "native" suffix to indicate that the output should be a representation 

93 of the native object type representing the header (which can be 

94 PropertyList or an Astropy header). Without this modify headers 

95 will be dumped as simple `dict` form. 

96 "auto" is used to indicate that a single file has been specified 

97 but the output will depend on whether the file is a multi-extension 

98 FITS file or not. 

99 write_heading : `bool`, optional 

100 If `True` and in table mode, write a table heading out before writing 

101 the content. 

102 

103 Returns 

104 ------- 

105 success : `bool` 

106 `True` if the file was handled successfully, `False` if the file 

107 could not be processed. 

108 """ 

109 if output_mode not in OUTPUT_MODES: 

110 raise ValueError(f"Output mode of '{output_mode}' is not understood.") 

111 

112 # This gets in the way in tabular mode 

113 if output_mode != "table": 

114 print(f"Analyzing {file}...", file=errstream) 

115 

116 try: 

117 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, can_raise=True) 

118 if md is None: 

119 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}") 

120 

121 if output_mode.endswith("native"): 

122 # Strip native and don't change type of md 

123 output_mode = output_mode[: -len("native")] 

124 else: 

125 # Rewrite md as simple dict for output 

126 md = {k: v for k, v in md.items()} 

127 

128 if output_mode in ("yaml", "fixed"): 

129 if output_mode == "fixed": 

130 fix_header(md, filename=file) 

131 

132 # The header should be written out in the insertion order 

133 print(yaml.dump(md, sort_keys=False), file=outstream) 

134 return True 

135 

136 # Try to work out a translator class. 

137 translator_class = MetadataTranslator.determine_translator(md, filename=file) 

138 

139 # Work out which headers to translate, assuming the default if 

140 # we have a YAML test file. 

141 if file.endswith(".yaml"): 

142 headers = [md] 

143 else: 

144 headers = list(translator_class.determine_translatable_headers(file, md)) 

145 if output_mode == "auto": 

146 output_mode = "table" if len(headers) > 1 else "verbose" 

147 

148 wrote_heading = False 

149 for md in headers: 

150 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

151 if output_mode == "table": 

152 columns = [ 

153 "{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) for c in TABLE_COLUMNS 

154 ] 

155 

156 if write_heading and not wrote_heading: 

157 # Construct headings of the same width as the items 

158 # we have calculated. Doing this means we don't have to 

159 # work out for ourselves how many characters will be used 

160 # for non-strings (especially Quantity) 

161 headings = [] 

162 separators = [] 

163 for thiscol, defn in zip(columns, TABLE_COLUMNS): 

164 width = len(thiscol) 

165 headings.append("{:{w}.{w}}".format(defn["label"], w=width)) 

166 separators.append("-" * width) 

167 print(" ".join(headings), file=outstream) 

168 print(" ".join(separators), file=outstream) 

169 wrote_heading = True 

170 

171 row = " ".join(columns) 

172 print(row, file=outstream) 

173 elif output_mode == "verbose": 

174 print(f"{obs_info}", file=outstream) 

175 elif output_mode == "none": 

176 pass 

177 else: 

178 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.") 

179 except Exception as e: 

180 if print_trace: 

181 traceback.print_exc(file=outstream) 

182 else: 

183 print(f"Failure processing {file}: {e}", file=outstream) 

184 return False 

185 return True 

186 

187 

188def translate_or_dump_headers( 

189 files: Sequence[str], 

190 regex: str, 

191 hdrnum: int, 

192 print_trace: bool, 

193 outstream: IO = sys.stdout, 

194 errstream: IO = sys.stderr, 

195 output_mode: str = "auto", 

196) -> tuple[list[str], list[str]]: 

197 """Read and translate metadata from the specified files. 

198 

199 Parameters 

200 ---------- 

201 files : iterable of `str` 

202 The files or directories from which the headers are to be read. 

203 regex : `str` 

204 Regular expression string used to filter files when a directory is 

205 scanned. 

206 hdrnum : `int` 

207 The HDU number to read. The primary header is always read and 

208 merged with the header from this HDU. 

209 print_trace : `bool` 

210 If there is an error reading the file and this parameter is `True`, 

211 a full traceback of the exception will be reported. If `False` prints 

212 a one line summary of the error condition. 

213 outstream : `io.StringIO`, optional 

214 Output stream to use for standard messages. Defaults to `sys.stdout`. 

215 errstream : `io.StringIO`, optional 

216 Stream to send messages that would normally be sent to standard 

217 error. Defaults to `sys.stderr`. 

218 output_mode : `str`, optional 

219 Output mode to use for the translated information. 

220 "auto" switches based on how many files are found. 

221 

222 Returns 

223 ------- 

224 okay : `list` of `str` 

225 All the files that were processed successfully. 

226 failed : `list` of `str` 

227 All the files that could not be processed. 

228 """ 

229 found_files = find_files(files, regex) 

230 

231 # Convert "auto" to correct mode but for a single file keep it 

232 # auto in case that file has multiple headers 

233 if output_mode == "auto": 

234 if len(found_files) > 1: 

235 output_mode = "table" 

236 

237 # Process each file 

238 failed = [] 

239 okay = [] 

240 heading = True 

241 for path in sorted(found_files): 

242 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode, heading) 

243 heading = False 

244 if isok: 

245 okay.append(path) 

246 else: 

247 failed.append(path) 

248 

249 return okay, failed