Coverage for python/astro_metadata_translator/bin/translateheader.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Implementation of the ``translate_header.py`` script.
14Read file metadata from the specified files and report the translated content.
15"""
17__all__ = ("main", "process_files")
19import argparse
20import logging
22import sys
23import traceback
24import importlib
25import yaml
26from astro_metadata_translator import ObservationInfo, fix_header, MetadataTranslator
28from ..file_helpers import find_files, read_basic_metadata_from_file
31# Output mode choices
32OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none")
34# Definitions for table columns
35TABLE_COLUMNS = ({
36 "format": "32.32s",
37 "attr": "observation_id",
38 "label": "ObsId"
39 },
40 {
41 "format": "8.8s",
42 "attr": "observation_type",
43 "label": "ImgType",
44 },
45 {
46 "format": "16.16s",
47 "attr": "object",
48 "label": "Object",
49 },
50 {
51 "format": "16.16s",
52 "attr": "physical_filter",
53 "label": "Filter",
54 },
55 {
56 "format": ">8.8s",
57 "attr": "detector_unique_name",
58 "label": "Detector"
59 },
60 {
61 "format": "5.1f",
62 "attr": "exposure_time",
63 "label": "ExpTime",
64 },
65 )
68def build_argparser():
69 """Construct an argument parser for the ``translate_header.py`` script.
71 Returns
72 -------
73 argparser : `argparse.ArgumentParser`
74 The argument parser that defines the ``translate_header.py``
75 command-line interface.
76 """
78 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files")
79 parser.add_argument("files", metavar="file", type=str, nargs="+",
80 help="File(s) from which headers will be parsed."
81 " If a directory is given it will be scanned for files matching the regular"
82 " expression defined in --regex.")
83 parser.add_argument("-q", "--quiet", action="store_true",
84 help="Do not report the translation content from each header. This forces "
85 "output mode 'none'.")
86 parser.add_argument("-d", "--dumphdr", action="store_true",
87 help="Dump the header in YAML format to standard output rather than translating it."
88 " This is the same as using mode=yaml")
89 parser.add_argument("--traceback", action="store_true",
90 help="Give detailed trace back when any errors encountered")
91 parser.add_argument("-n", "--hdrnum", default=1,
92 help="HDU number to read. If the HDU can not be found, a warning is issued but "
93 "translation is attempted using the primary header. "
94 "The primary header is always read and merged with this header.")
95 parser.add_argument("-m", "--mode", default="auto", choices=OUTPUT_MODES,
96 help="Display mode for translated parameters. 'verbose' displays all the information"
97 " available. 'table' displays important information in tabular form."
98 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)."
99 " 'fixed' dumps the header in YAML after it has had corrections applied."
100 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form."
101 " 'none' displays no translated header information and is an alias for the "
102 " '--quiet' option."
103 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.")
104 parser.add_argument("-l", "--log", default="warn",
105 help="Python logging level to use.")
107 re_default = r"\.fit[s]?\b"
108 parser.add_argument("-r", "--regex", default=re_default,
109 help="When looking in a directory, regular expression to use to determine whether"
110 f" a file should be examined. Default: '{re_default}'")
112 parser.add_argument("-p", "--packages", action="append", type=str,
113 help="Python packages to import to register additional translators")
115 return parser
118def read_file(file, hdrnum, print_trace,
119 outstream=sys.stdout, errstream=sys.stderr, output_mode="verbose",
120 write_heading=False):
121 """Read the specified file and process it.
123 Parameters
124 ----------
125 file : `str`
126 The file from which the header is to be read.
127 hdrnum : `int`
128 The HDU number to read. The primary header is always read and
129 merged with the header from this HDU.
130 print_trace : `bool`
131 If there is an error reading the file and this parameter is `True`,
132 a full traceback of the exception will be reported. If `False` prints
133 a one line summary of the error condition.
134 outstream : `io.StringIO`, optional
135 Output stream to use for standard messages. Defaults to `sys.stdout`.
136 errstream : `io.StringIO`, optional
137 Stream to send messages that would normally be sent to standard
138 error. Defaults to `sys.stderr`.
139 output_mode : `str`, optional
140 Output mode to use. Must be one of "verbose", "none", "table",
141 "yaml", or "fixed". "yaml" and "fixed" can be modified with a
142 "native" suffix to indicate that the output should be a representation
143 of the native object type representing the header (which can be
144 PropertyList or an Astropy header). Without this modify headers
145 will be dumped as simple `dict` form.
146 "auto" is used to indicate that a single file has been specified
147 but the output will depend on whether the file is a multi-extension
148 FITS file or not.
149 write_heading: `bool`, optional
150 If `True` and in table mode, write a table heading out before writing
151 the content.
153 Returns
154 -------
155 success : `bool`
156 `True` if the file was handled successfully, `False` if the file
157 could not be processed.
158 """
159 if output_mode not in OUTPUT_MODES:
160 raise ValueError(f"Output mode of '{output_mode}' is not understood.")
162 # This gets in the way in tabular mode
163 if output_mode != "table":
164 print(f"Analyzing {file}...", file=errstream)
166 try:
167 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, can_raise=True)
168 if md is None:
169 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}")
171 if output_mode.endswith("native"):
172 # Strip native and don't change type of md
173 output_mode = output_mode[:-len("native")]
174 else:
175 # Rewrite md as simple dict for output
176 md = {k: v for k, v in md.items()}
178 if output_mode in ("yaml", "fixed"):
180 if output_mode == "fixed":
181 fix_header(md, filename=file)
183 # The header should be written out in the insertion order
184 print(yaml.dump(md, sort_keys=False), file=outstream)
185 return True
187 # Try to work out a translator class.
188 translator_class = MetadataTranslator.determine_translator(md, filename=file)
190 # Work out which headers to translate, assuming the default if
191 # we have a YAML test file.
192 if file.endswith(".yaml"):
193 headers = [md]
194 else:
195 headers = list(translator_class.determine_translatable_headers(file, md))
196 if output_mode == "auto":
197 output_mode = "table" if len(headers) > 1 else "verbose"
199 wrote_heading = False
200 for md in headers:
201 obs_info = ObservationInfo(md, pedantic=True, filename=file)
202 if output_mode == "table":
203 columns = ["{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"])
204 for c in TABLE_COLUMNS]
206 if write_heading and not wrote_heading:
207 # Construct headings of the same width as the items
208 # we have calculated. Doing this means we don't have to
209 # work out for ourselves how many characters will be used
210 # for non-strings (especially Quantity)
211 headings = []
212 separators = []
213 for thiscol, defn in zip(columns, TABLE_COLUMNS):
214 width = len(thiscol)
215 headings.append("{:{w}.{w}}".format(defn["label"], w=width))
216 separators.append("-"*width)
217 print(" ".join(headings), file=outstream)
218 print(" ".join(separators), file=outstream)
219 wrote_heading = True
221 row = " ".join(columns)
222 print(row, file=outstream)
223 elif output_mode == "verbose":
224 print(f"{obs_info}", file=outstream)
225 elif output_mode == "none":
226 pass
227 else:
228 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.")
229 except Exception as e:
230 if print_trace:
231 traceback.print_exc(file=outstream)
232 else:
233 print(f"Failure processing {file}: {e}", file=outstream)
234 return False
235 return True
238def process_files(files, regex, hdrnum, print_trace,
239 outstream=sys.stdout, errstream=sys.stderr,
240 output_mode="auto"):
241 """Read and translate metadata from the specified files.
243 Parameters
244 ----------
245 files : iterable of `str`
246 The files or directories from which the headers are to be read.
247 regex : `str`
248 Regular expression string used to filter files when a directory is
249 scanned.
250 hdrnum : `int`
251 The HDU number to read. The primary header is always read and
252 merged with the header from this HDU.
253 print_trace : `bool`
254 If there is an error reading the file and this parameter is `True`,
255 a full traceback of the exception will be reported. If `False` prints
256 a one line summary of the error condition.
257 outstream : `io.StringIO`, optional
258 Output stream to use for standard messages. Defaults to `sys.stdout`.
259 errstream : `io.StringIO`, optional
260 Stream to send messages that would normally be sent to standard
261 error. Defaults to `sys.stderr`.
262 output_mode : `str`, optional
263 Output mode to use for the translated information.
264 "auto" switches based on how many files are found.
266 Returns
267 -------
268 okay : `list` of `str`
269 All the files that were processed successfully.
270 failed : `list` of `str`
271 All the files that could not be processed.
272 """
273 found_files = find_files(files, regex)
275 # Convert "auto" to correct mode but for a single file keep it
276 # auto in case that file has multiple headers
277 if output_mode == "auto":
278 if len(found_files) > 1:
279 output_mode = "table"
281 # Process each file
282 failed = []
283 okay = []
284 heading = True
285 for path in sorted(found_files):
286 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode,
287 heading)
288 heading = False
289 if isok:
290 okay.append(path)
291 else:
292 failed.append(path)
294 return okay, failed
297def main():
298 """Read metadata from the supplied files and translate the content to
299 standard form.
301 Returns
302 -------
303 status : `int`
304 Exit status to be passed to `sys.exit()`. 0 if any of the files
305 could be translated. 1 otherwise.
306 """
308 logging.warn("This command is deprecated. Please use 'astrometadata translate' "
309 " or 'astrometadata dump' instead. See 'astrometadata -h' for more details.")
311 args = build_argparser().parse_args()
313 # Process import requests
314 if args.packages:
315 for m in args.packages:
316 importlib.import_module(m)
318 output_mode = args.mode
319 if args.quiet:
320 output_mode = "none"
321 elif args.dumphdr:
322 output_mode = "yaml"
324 # Set the log level. Convert to upper case to allow the user to
325 # specify --log=DEBUG or --log=debug
326 numeric_level = getattr(logging, args.log.upper(), None)
327 if not isinstance(numeric_level, int):
328 raise ValueError(f"Invalid log level: {args.log}")
329 logging.basicConfig(level=numeric_level)
331 # Main loop over files
332 okay, failed = process_files(args.files, args.regex, args.hdrnum,
333 args.traceback,
334 output_mode=output_mode)
336 if failed:
337 print("Files with failed translations:", file=sys.stderr)
338 for f in failed:
339 print(f"\t{f}", file=sys.stderr)
341 if okay:
342 # Good status if anything was returned in okay
343 return 0
344 else:
345 return 1