Coverage for python/astro_metadata_translator/bin/translate.py: 12%

79 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-06 03:48 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Implementation of the ``astrometadata translate`` command-line. 

13 

14Read file metadata from the specified files and report the translated content. 

15""" 

16 

17from __future__ import annotations 

18 

19__all__ = ("translate_or_dump_headers",) 

20 

21import logging 

22import traceback 

23from collections.abc import Sequence 

24from typing import IO 

25 

26import yaml 

27 

28from astro_metadata_translator import MetadataTranslator, ObservationInfo, fix_header 

29 

30from ..file_helpers import find_files, read_basic_metadata_from_file 

31 

32log = logging.getLogger(__name__) 

33 

34# Output mode choices 

35OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none") 

36 

37# Definitions for table columns 

38TABLE_COLUMNS = ( 

39 {"format": "32.32s", "attr": "observation_id", "label": "ObsId"}, 

40 { 

41 "format": "8.8s", 

42 "attr": "observation_type", 

43 "label": "ImgType", 

44 }, 

45 { 

46 "format": "16.16s", 

47 "attr": "object", 

48 "label": "Object", 

49 }, 

50 { 

51 "format": "16.16s", 

52 "attr": "physical_filter", 

53 "label": "Filter", 

54 }, 

55 {"format": ">8.8s", "attr": "detector_unique_name", "label": "Detector"}, 

56 { 

57 "format": "5.1f", 

58 "attr": "exposure_time", 

59 "label": "ExpTime", 

60 }, 

61) 

62 

63 

64def read_file( 

65 file: str, 

66 hdrnum: int, 

67 print_trace: bool, 

68 outstream: IO | None = None, 

69 output_mode: str = "verbose", 

70 write_heading: bool = False, 

71) -> bool: 

72 """Read the specified file and process it. 

73 

74 Parameters 

75 ---------- 

76 file : `str` 

77 The file from which the header is to be read. 

78 hdrnum : `int` 

79 The HDU number to read. The primary header is always read and 

80 merged with the header from this HDU. 

81 print_trace : `bool` 

82 If there is an error reading the file and this parameter is `True`, 

83 a full traceback of the exception will be reported. If `False` prints 

84 a one line summary of the error condition. 

85 outstream : `io.StringIO`, optional 

86 Output stream to use for standard messages. Defaults to `None` which 

87 uses the default output stream. 

88 output_mode : `str`, optional 

89 Output mode to use. Must be one of "verbose", "none", "table", 

90 "yaml", or "fixed". "yaml" and "fixed" can be modified with a 

91 "native" suffix to indicate that the output should be a representation 

92 of the native object type representing the header (which can be 

93 PropertyList or an Astropy header). Without this modify headers 

94 will be dumped as simple `dict` form. 

95 "auto" is used to indicate that a single file has been specified 

96 but the output will depend on whether the file is a multi-extension 

97 FITS file or not. 

98 write_heading : `bool`, optional 

99 If `True` and in table mode, write a table heading out before writing 

100 the content. 

101 

102 Returns 

103 ------- 

104 success : `bool` 

105 `True` if the file was handled successfully, `False` if the file 

106 could not be processed. 

107 """ 

108 if output_mode not in OUTPUT_MODES: 

109 raise ValueError(f"Output mode of '{output_mode}' is not understood.") 

110 

111 # This gets in the way in tabular mode 

112 if output_mode != "table": 

113 log.info("Analyzing %s...", file) 

114 

115 try: 

116 md = read_basic_metadata_from_file(file, hdrnum, can_raise=True) 

117 if md is None: 

118 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}") 

119 

120 if output_mode.endswith("native"): 

121 # Strip native and don't change type of md 

122 output_mode = output_mode[: -len("native")] 

123 else: 

124 # Rewrite md as simple dict for output 

125 md = {k: v for k, v in md.items()} 

126 

127 if output_mode in ("yaml", "fixed"): 

128 if output_mode == "fixed": 

129 fix_header(md, filename=file) 

130 

131 # The header should be written out in the insertion order 

132 print(yaml.dump(md, sort_keys=False), file=outstream) 

133 return True 

134 

135 # Try to work out a translator class. 

136 translator_class = MetadataTranslator.determine_translator(md, filename=file) 

137 

138 # Work out which headers to translate, assuming the default if 

139 # we have a YAML test file. 

140 if file.endswith(".yaml"): 

141 headers = [md] 

142 else: 

143 headers = list(translator_class.determine_translatable_headers(file, md)) 

144 if output_mode == "auto": 

145 output_mode = "table" if len(headers) > 1 else "verbose" 

146 

147 wrote_heading = False 

148 for md in headers: 

149 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

150 if output_mode == "table": 

151 columns = [ 

152 "{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) for c in TABLE_COLUMNS 

153 ] 

154 

155 if write_heading and not wrote_heading: 

156 # Construct headings of the same width as the items 

157 # we have calculated. Doing this means we don't have to 

158 # work out for ourselves how many characters will be used 

159 # for non-strings (especially Quantity) 

160 headings = [] 

161 separators = [] 

162 for thiscol, defn in zip(columns, TABLE_COLUMNS): 

163 width = len(thiscol) 

164 headings.append("{:{w}.{w}}".format(defn["label"], w=width)) 

165 separators.append("-" * width) 

166 print(" ".join(headings), file=outstream) 

167 print(" ".join(separators), file=outstream) 

168 wrote_heading = True 

169 

170 row = " ".join(columns) 

171 print(row, file=outstream) 

172 elif output_mode == "verbose": 

173 print(f"{obs_info}", file=outstream) 

174 elif output_mode == "none": 

175 pass 

176 else: 

177 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.") 

178 except Exception as e: 

179 if print_trace: 

180 traceback.print_exc(file=outstream) 

181 else: 

182 print(f"Failure processing {file}: {e}", file=outstream) 

183 return False 

184 return True 

185 

186 

187def translate_or_dump_headers( 

188 files: Sequence[str], 

189 regex: str, 

190 hdrnum: int, 

191 print_trace: bool, 

192 outstream: IO | None = None, 

193 output_mode: str = "auto", 

194) -> tuple[list[str], list[str]]: 

195 """Read and translate metadata from the specified files. 

196 

197 Parameters 

198 ---------- 

199 files : iterable of `str` 

200 The files or directories from which the headers are to be read. 

201 regex : `str` 

202 Regular expression string used to filter files when a directory is 

203 scanned. 

204 hdrnum : `int` 

205 The HDU number to read. The primary header is always read and 

206 merged with the header from this HDU. 

207 print_trace : `bool` 

208 If there is an error reading the file and this parameter is `True`, 

209 a full traceback of the exception will be reported. If `False` prints 

210 a one line summary of the error condition. 

211 outstream : `io.StringIO` or `None`, optional 

212 Output stream to use for standard messages. Defaults to `None` which 

213 uses the default output stream. 

214 output_mode : `str`, optional 

215 Output mode to use for the translated information. 

216 "auto" switches based on how many files are found. 

217 

218 Returns 

219 ------- 

220 okay : `list` of `str` 

221 All the files that were processed successfully. 

222 failed : `list` of `str` 

223 All the files that could not be processed. 

224 """ 

225 found_files = find_files(files, regex) 

226 

227 # Convert "auto" to correct mode but for a single file keep it 

228 # auto in case that file has multiple headers 

229 if output_mode == "auto": 

230 if len(found_files) > 1: 

231 output_mode = "table" 

232 

233 # Process each file 

234 failed = [] 

235 okay = [] 

236 heading = True 

237 for path in sorted(found_files): 

238 isok = read_file(path, hdrnum, print_trace, outstream, output_mode, heading) 

239 heading = False 

240 if isok: 

241 okay.append(path) 

242 else: 

243 failed.append(path) 

244 

245 return okay, failed