Coverage for python/astro_metadata_translator/bin/translateheader.py: 10%
116 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-07 09:25 +0000
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-07 09:25 +0000
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Implementation of the ``translate_header.py`` script.
14Read file metadata from the specified files and report the translated content.
15"""
17from __future__ import annotations
19__all__ = ("main", "process_files")
21import argparse
22import importlib
23import logging
24import sys
25import traceback
26from typing import IO, List, Sequence, Tuple
28import yaml
30from astro_metadata_translator import MetadataTranslator, ObservationInfo, fix_header
32from ..file_helpers import find_files, read_basic_metadata_from_file
34# Output mode choices
35OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none")
37# Definitions for table columns
38TABLE_COLUMNS = (
39 {"format": "32.32s", "attr": "observation_id", "label": "ObsId"},
40 {
41 "format": "8.8s",
42 "attr": "observation_type",
43 "label": "ImgType",
44 },
45 {
46 "format": "16.16s",
47 "attr": "object",
48 "label": "Object",
49 },
50 {
51 "format": "16.16s",
52 "attr": "physical_filter",
53 "label": "Filter",
54 },
55 {"format": ">8.8s", "attr": "detector_unique_name", "label": "Detector"},
56 {
57 "format": "5.1f",
58 "attr": "exposure_time",
59 "label": "ExpTime",
60 },
61)
64def build_argparser() -> argparse.ArgumentParser:
65 """Construct an argument parser for the ``translate_header.py`` script.
67 Returns
68 -------
69 argparser : `argparse.ArgumentParser`
70 The argument parser that defines the ``translate_header.py``
71 command-line interface.
72 """
74 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files")
75 parser.add_argument(
76 "files",
77 metavar="file",
78 type=str,
79 nargs="+",
80 help="File(s) from which headers will be parsed."
81 " If a directory is given it will be scanned for files matching the regular"
82 " expression defined in --regex.",
83 )
84 parser.add_argument(
85 "-q",
86 "--quiet",
87 action="store_true",
88 help="Do not report the translation content from each header. This forces output mode 'none'.",
89 )
90 parser.add_argument(
91 "-d",
92 "--dumphdr",
93 action="store_true",
94 help="Dump the header in YAML format to standard output rather than translating it."
95 " This is the same as using mode=yaml",
96 )
97 parser.add_argument(
98 "--traceback", action="store_true", help="Give detailed trace back when any errors encountered"
99 )
100 parser.add_argument(
101 "-n",
102 "--hdrnum",
103 default=1,
104 help="HDU number to read. If the HDU can not be found, a warning is issued but "
105 "translation is attempted using the primary header. "
106 "The primary header is always read and merged with this header.",
107 )
108 parser.add_argument(
109 "-m",
110 "--mode",
111 default="auto",
112 choices=OUTPUT_MODES,
113 help="Display mode for translated parameters. 'verbose' displays all the information"
114 " available. 'table' displays important information in tabular form."
115 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)."
116 " 'fixed' dumps the header in YAML after it has had corrections applied."
117 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form."
118 " 'none' displays no translated header information and is an alias for the "
119 " '--quiet' option."
120 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.",
121 )
122 parser.add_argument("-l", "--log", default="warn", help="Python logging level to use.")
124 re_default = r"\.fit[s]?\b"
125 parser.add_argument(
126 "-r",
127 "--regex",
128 default=re_default,
129 help="When looking in a directory, regular expression to use to determine whether"
130 f" a file should be examined. Default: '{re_default}'",
131 )
133 parser.add_argument(
134 "-p",
135 "--packages",
136 action="append",
137 type=str,
138 help="Python packages to import to register additional translators",
139 )
141 return parser
144def read_file(
145 file: str,
146 hdrnum: int,
147 print_trace: bool,
148 outstream: IO = sys.stdout,
149 errstream: IO = sys.stderr,
150 output_mode: str = "verbose",
151 write_heading: bool = False,
152) -> bool:
153 """Read the specified file and process it.
155 Parameters
156 ----------
157 file : `str`
158 The file from which the header is to be read.
159 hdrnum : `int`
160 The HDU number to read. The primary header is always read and
161 merged with the header from this HDU.
162 print_trace : `bool`
163 If there is an error reading the file and this parameter is `True`,
164 a full traceback of the exception will be reported. If `False` prints
165 a one line summary of the error condition.
166 outstream : `io.StringIO`, optional
167 Output stream to use for standard messages. Defaults to `sys.stdout`.
168 errstream : `io.StringIO`, optional
169 Stream to send messages that would normally be sent to standard
170 error. Defaults to `sys.stderr`.
171 output_mode : `str`, optional
172 Output mode to use. Must be one of "verbose", "none", "table",
173 "yaml", or "fixed". "yaml" and "fixed" can be modified with a
174 "native" suffix to indicate that the output should be a representation
175 of the native object type representing the header (which can be
176 PropertyList or an Astropy header). Without this modify headers
177 will be dumped as simple `dict` form.
178 "auto" is used to indicate that a single file has been specified
179 but the output will depend on whether the file is a multi-extension
180 FITS file or not.
181 write_heading: `bool`, optional
182 If `True` and in table mode, write a table heading out before writing
183 the content.
185 Returns
186 -------
187 success : `bool`
188 `True` if the file was handled successfully, `False` if the file
189 could not be processed.
190 """
191 if output_mode not in OUTPUT_MODES:
192 raise ValueError(f"Output mode of '{output_mode}' is not understood.")
194 # This gets in the way in tabular mode
195 if output_mode != "table":
196 print(f"Analyzing {file}...", file=errstream)
198 try:
199 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, can_raise=True)
200 if md is None:
201 raise RuntimeError(f"Failed to read file {file} HDU={hdrnum}")
203 if output_mode.endswith("native"):
204 # Strip native and don't change type of md
205 output_mode = output_mode[: -len("native")]
206 else:
207 # Rewrite md as simple dict for output
208 md = {k: v for k, v in md.items()}
210 if output_mode in ("yaml", "fixed"):
212 if output_mode == "fixed":
213 fix_header(md, filename=file)
215 # The header should be written out in the insertion order
216 print(yaml.dump(md, sort_keys=False), file=outstream)
217 return True
219 # Try to work out a translator class.
220 translator_class = MetadataTranslator.determine_translator(md, filename=file)
222 # Work out which headers to translate, assuming the default if
223 # we have a YAML test file.
224 if file.endswith(".yaml"):
225 headers = [md]
226 else:
227 headers = list(translator_class.determine_translatable_headers(file, md))
228 if output_mode == "auto":
229 output_mode = "table" if len(headers) > 1 else "verbose"
231 wrote_heading = False
232 for md in headers:
233 obs_info = ObservationInfo(md, pedantic=True, filename=file)
234 if output_mode == "table":
235 columns = [
236 "{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"]) for c in TABLE_COLUMNS
237 ]
239 if write_heading and not wrote_heading:
240 # Construct headings of the same width as the items
241 # we have calculated. Doing this means we don't have to
242 # work out for ourselves how many characters will be used
243 # for non-strings (especially Quantity)
244 headings = []
245 separators = []
246 for thiscol, defn in zip(columns, TABLE_COLUMNS):
247 width = len(thiscol)
248 headings.append("{:{w}.{w}}".format(defn["label"], w=width))
249 separators.append("-" * width)
250 print(" ".join(headings), file=outstream)
251 print(" ".join(separators), file=outstream)
252 wrote_heading = True
254 row = " ".join(columns)
255 print(row, file=outstream)
256 elif output_mode == "verbose":
257 print(f"{obs_info}", file=outstream)
258 elif output_mode == "none":
259 pass
260 else:
261 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.")
262 except Exception as e:
263 if print_trace:
264 traceback.print_exc(file=outstream)
265 else:
266 print(f"Failure processing {file}: {e}", file=outstream)
267 return False
268 return True
271def process_files(
272 files: Sequence[str],
273 regex: str,
274 hdrnum: int,
275 print_trace: bool,
276 outstream: IO = sys.stdout,
277 errstream: IO = sys.stderr,
278 output_mode: str = "auto",
279) -> Tuple[List[str], List[str]]:
280 """Read and translate metadata from the specified files.
282 Parameters
283 ----------
284 files : iterable of `str`
285 The files or directories from which the headers are to be read.
286 regex : `str`
287 Regular expression string used to filter files when a directory is
288 scanned.
289 hdrnum : `int`
290 The HDU number to read. The primary header is always read and
291 merged with the header from this HDU.
292 print_trace : `bool`
293 If there is an error reading the file and this parameter is `True`,
294 a full traceback of the exception will be reported. If `False` prints
295 a one line summary of the error condition.
296 outstream : `io.StringIO`, optional
297 Output stream to use for standard messages. Defaults to `sys.stdout`.
298 errstream : `io.StringIO`, optional
299 Stream to send messages that would normally be sent to standard
300 error. Defaults to `sys.stderr`.
301 output_mode : `str`, optional
302 Output mode to use for the translated information.
303 "auto" switches based on how many files are found.
305 Returns
306 -------
307 okay : `list` of `str`
308 All the files that were processed successfully.
309 failed : `list` of `str`
310 All the files that could not be processed.
311 """
312 found_files = find_files(files, regex)
314 # Convert "auto" to correct mode but for a single file keep it
315 # auto in case that file has multiple headers
316 if output_mode == "auto":
317 if len(found_files) > 1:
318 output_mode = "table"
320 # Process each file
321 failed = []
322 okay = []
323 heading = True
324 for path in sorted(found_files):
325 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode, heading)
326 heading = False
327 if isok:
328 okay.append(path)
329 else:
330 failed.append(path)
332 return okay, failed
335def main() -> int:
336 """Read metadata from the supplied files and translate the content to
337 standard form.
339 Returns
340 -------
341 status : `int`
342 Exit status to be passed to `sys.exit()`. 0 if any of the files
343 could be translated. 1 otherwise.
344 """
346 logging.warn(
347 "This command is deprecated. Please use 'astrometadata translate' "
348 " or 'astrometadata dump' instead. See 'astrometadata -h' for more details."
349 )
351 args = build_argparser().parse_args()
353 # Process import requests
354 if args.packages:
355 for m in args.packages:
356 importlib.import_module(m)
358 output_mode = args.mode
359 if args.quiet:
360 output_mode = "none"
361 elif args.dumphdr:
362 output_mode = "yaml"
364 # Set the log level. Convert to upper case to allow the user to
365 # specify --log=DEBUG or --log=debug
366 numeric_level = getattr(logging, args.log.upper(), None)
367 if not isinstance(numeric_level, int):
368 raise ValueError(f"Invalid log level: {args.log}")
369 logging.basicConfig(level=numeric_level)
371 # Main loop over files
372 okay, failed = process_files(args.files, args.regex, args.hdrnum, args.traceback, output_mode=output_mode)
374 if failed:
375 print("Files with failed translations:", file=sys.stderr)
376 for f in failed:
377 print(f"\t{f}", file=sys.stderr)
379 if okay:
380 # Good status if anything was returned in okay
381 return 0
382 else:
383 return 1