Coverage for python/astro_metadata_translator/bin/translateheader.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Implementation of the ``translate_header.py`` script.
14Read file metadata from the specified files and report the translated content.
15"""
17__all__ = ("main", "process_files")
19import argparse
20import logging
22import os
23import re
24import sys
25import traceback
26import importlib
27import yaml
28from astro_metadata_translator import ObservationInfo, merge_headers, fix_header
29from astro_metadata_translator.tests import read_test_file
31# Prefer afw over Astropy
32try:
33 from lsst.afw.fits import readMetadata
34 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata
36 def read_metadata(file, hdu):
37 try:
38 return readMetadata(file, hdu=hdu)
39 except lsst.afw.fits.FitsError:
40 return None
42except ImportError:
43 from astropy.io import fits
45 def read_metadata(file, hdu):
46 fits_file = fits.open(file)
47 try:
48 header = fits_file[hdu].header
49 except IndexError:
50 header = None
51 return header
54# Output mode choices
55OUTPUT_MODES = ("auto", "verbose", "table", "yaml", "fixed", "yamlnative", "fixednative", "none")
57# Definitions for table columns
58TABLE_COLUMNS = ({
59 "format": "32.32s",
60 "attr": "observation_id",
61 "label": "ObsId"
62 },
63 {
64 "format": "8.8s",
65 "attr": "observation_type",
66 "label": "ImgType",
67 },
68 {
69 "format": "16.16s",
70 "attr": "object",
71 "label": "Object",
72 },
73 {
74 "format": "16.16s",
75 "attr": "physical_filter",
76 "label": "Filter",
77 },
78 {
79 "format": "5.1f",
80 "attr": "exposure_time",
81 "label": "ExpTime",
82 },
83 )
86def build_argparser():
87 """Construct an argument parser for the ``translate_header.py`` script.
89 Returns
90 -------
91 argparser : `argparse.ArgumentParser`
92 The argument parser that defines the ``translate_header.py``
93 command-line interface.
94 """
96 parser = argparse.ArgumentParser(description="Summarize headers from astronomical data files")
97 parser.add_argument("files", metavar="file", type=str, nargs="+",
98 help="File(s) from which headers will be parsed."
99 " If a directory is given it will be scanned for files matching the regular"
100 " expression defined in --regex.")
101 parser.add_argument("-q", "--quiet", action="store_true",
102 help="Do not report the translation content from each header. This forces "
103 "output mode 'none'.")
104 parser.add_argument("-d", "--dumphdr", action="store_true",
105 help="Dump the header in YAML format to standard output rather than translating it."
106 " This is the same as using mode=yaml")
107 parser.add_argument("--traceback", action="store_true",
108 help="Give detailed trace back when any errors encountered")
109 parser.add_argument("-n", "--hdrnum", default=1,
110 help="HDU number to read. If the HDU can not be found, a warning is issued but "
111 "translation is attempted using the primary header. "
112 "The primary header is always read and merged with this header.")
113 parser.add_argument("-m", "--mode", default="auto", choices=OUTPUT_MODES,
114 help="Display mode for translated parameters. 'verbose' displays all the information"
115 " available. 'table' displays important information in tabular form."
116 " 'yaml' dumps the header in YAML format (this is equivalent to -d option)."
117 " 'fixed' dumps the header in YAML after it has had corrections applied."
118 " Add 'native' suffix to dump YAML in PropertyList or Astropy native form."
119 " 'none' displays no translated header information and is an alias for the "
120 " '--quiet' option."
121 " 'auto' mode is 'verbose' for a single file and 'table' for multiple files.")
122 parser.add_argument("-l", "--log", default="warn",
123 help="Python logging level to use.")
125 re_default = r"\.fit[s]?\b"
126 parser.add_argument("-r", "--regex", default=re_default,
127 help="When looking in a directory, regular expression to use to determine whether"
128 f" a file should be examined. Default: '{re_default}'")
130 parser.add_argument("-p", "--packages", action="append", type=str,
131 help="Python packages to import to register additional translators")
133 return parser
136def read_file(file, hdrnum, print_trace,
137 outstream=sys.stdout, errstream=sys.stderr, output_mode="verbose",
138 write_heading=False):
139 """Read the specified file and process it.
141 Parameters
142 ----------
143 file : `str`
144 The file from which the header is to be read.
145 hdrnum : `int`
146 The HDU number to read. The primary header is always read and
147 merged with the header from this HDU.
148 print_trace : `bool`
149 If there is an error reading the file and this parameter is `True`,
150 a full traceback of the exception will be reported. If `False` prints
151 a one line summary of the error condition.
152 outstream : `io.StringIO`, optional
153 Output stream to use for standard messages. Defaults to `sys.stdout`.
154 errstream : `io.StringIO`, optional
155 Stream to send messages that would normally be sent to standard
156 error. Defaults to `sys.stderr`.
157 output_mode : `str`, optional
158 Output mode to use. Must be one of "verbose", "none", "table",
159 "yaml", or "fixed". "yaml" and "fixed" can be modified with a
160 "native" suffix to indicate that the output should be a representation
161 of the native object type representing the header (which can be
162 PropertyList or an Astropy header). Without this modify headers
163 will be dumped as simple `dict` form.
164 "auto" is not allowed by this point.
165 write_heading: `bool`, optional
166 If `True` and in table mode, write a table heading out before writing
167 the content.
169 Returns
170 -------
171 success : `bool`
172 `True` if the file was handled successfully, `False` if the file
173 could not be processed.
174 """
175 if output_mode not in OUTPUT_MODES:
176 raise ValueError(f"Output mode of '{output_mode}' is not understood.")
177 if output_mode == "auto":
178 raise ValueError("Output mode can not be 'auto' here.")
180 # This gets in the way in tabular mode
181 if output_mode != "table":
182 print(f"Analyzing {file}...", file=errstream)
184 try:
185 if file.endswith(".yaml"):
186 md = read_test_file(file,)
187 if hdrnum != 0:
188 # YAML can't have HDUs
189 hdrnum = 0
190 else:
191 md = read_metadata(file, 0)
192 if md is None:
193 print(f"Unable to open file {file}", file=errstream)
194 return False
195 if hdrnum != 0:
196 mdn = read_metadata(file, int(hdrnum))
197 # Astropy does not allow append mode since it does not
198 # convert lists to multiple cards. Overwrite for now
199 if mdn is not None:
200 md = merge_headers([md, mdn], mode="overwrite")
201 else:
202 print(f"HDU {hdrnum} was not found. Ignoring request.", file=errstream)
204 if output_mode.endswith("native"):
205 # Strip native and don't change type of md
206 output_mode = output_mode[:-len("native")]
207 else:
208 # Rewrite md as simple dict for output
209 md = {k: v for k, v in md.items()}
211 if output_mode in ("yaml", "fixed"):
213 if output_mode == "fixed":
214 fix_header(md)
216 # The header should be written out in the insertion order
217 print(yaml.dump(md, sort_keys=False), file=outstream)
218 return True
219 obs_info = ObservationInfo(md, pedantic=True, filename=file)
220 if output_mode == "table":
221 columns = ["{:{fmt}}".format(getattr(obs_info, c["attr"]), fmt=c["format"])
222 for c in TABLE_COLUMNS]
224 if write_heading:
225 # Construct headings of the same width as the items
226 # we have calculated. Doing this means we don't have to
227 # work out for ourselves how many characters will be used
228 # for non-strings (especially Quantity)
229 headings = []
230 separators = []
231 for thiscol, defn in zip(columns, TABLE_COLUMNS):
232 width = len(thiscol)
233 headings.append("{:{w}.{w}}".format(defn["label"], w=width))
234 separators.append("-"*width)
235 print(" ".join(headings), file=outstream)
236 print(" ".join(separators), file=outstream)
238 row = " ".join(columns)
239 print(row, file=outstream)
240 elif output_mode == "verbose":
241 print(f"{obs_info}", file=outstream)
242 elif output_mode == "none":
243 pass
244 else:
245 raise RuntimeError(f"Output mode of '{output_mode}' not recognized but should be known.")
246 except Exception as e:
247 if print_trace:
248 traceback.print_exc(file=outstream)
249 else:
250 print(repr(e), file=outstream)
251 return False
252 return True
255def process_files(files, regex, hdrnum, print_trace,
256 outstream=sys.stdout, errstream=sys.stderr,
257 output_mode="auto"):
258 """Read and translate metadata from the specified files.
260 Parameters
261 ----------
262 files : iterable of `str`
263 The files or directories from which the headers are to be read.
264 regex : `str`
265 Regular expression string used to filter files when a directory is
266 scanned.
267 hdrnum : `int`
268 The HDU number to read. The primary header is always read and
269 merged with the header from this HDU.
270 print_trace : `bool`
271 If there is an error reading the file and this parameter is `True`,
272 a full traceback of the exception will be reported. If `False` prints
273 a one line summary of the error condition.
274 outstream : `io.StringIO`, optional
275 Output stream to use for standard messages. Defaults to `sys.stdout`.
276 errstream : `io.StringIO`, optional
277 Stream to send messages that would normally be sent to standard
278 error. Defaults to `sys.stderr`.
279 output_mode : `str`, optional
280 Output mode to use for the translated information.
281 "auto" switches based on how many files are found.
283 Returns
284 -------
285 okay : `list` of `str`
286 All the files that were processed successfully.
287 failed : `list` of `str`
288 All the files that could not be processed.
289 """
290 file_regex = re.compile(regex)
291 found_files = []
293 # Find all the files of interest
294 for file in files:
295 if os.path.isdir(file):
296 for root, dirs, files in os.walk(file):
297 for name in files:
298 path = os.path.join(root, name)
299 if os.path.isfile(path) and file_regex.search(name):
300 found_files.append(path)
301 else:
302 found_files.append(file)
304 # Convert "auto" to correct mode
305 if output_mode == "auto":
306 if len(found_files) > 1:
307 output_mode = "table"
308 else:
309 output_mode = "verbose"
311 # Process each file
312 failed = []
313 okay = []
314 heading = True
315 for path in sorted(found_files):
316 isok = read_file(path, hdrnum, print_trace, outstream, errstream, output_mode,
317 heading)
318 heading = False
319 if isok:
320 okay.append(path)
321 else:
322 failed.append(path)
324 return okay, failed
327def main():
328 """Read metadata from the supplied files and translate the content to
329 standard form.
331 Returns
332 -------
333 status : `int`
334 Exit status to be passed to `sys.exit()`. 0 if any of the files
335 could be translated. 1 otherwise.
336 """
337 args = build_argparser().parse_args()
339 # Process import requests
340 if args.packages:
341 for m in args.packages:
342 importlib.import_module(m)
344 output_mode = args.mode
345 if args.quiet:
346 output_mode = "none"
347 elif args.dumphdr:
348 output_mode = "yaml"
350 # Set the log level. Convert to upper case to allow the user to
351 # specify --log=DEBUG or --log=debug
352 numeric_level = getattr(logging, args.log.upper(), None)
353 if not isinstance(numeric_level, int):
354 raise ValueError(f"Invalid log level: {args.log}")
355 logging.basicConfig(level=numeric_level)
357 # Main loop over files
358 okay, failed = process_files(args.files, args.regex, args.hdrnum,
359 args.traceback,
360 output_mode=output_mode)
362 if failed:
363 print("Files with failed translations:", file=sys.stderr)
364 for f in failed:
365 print(f"\t{f}", file=sys.stderr)
367 if okay:
368 # Good status if anything was returned in okay
369 return 0
370 else:
371 return 1