Coverage for python/astro_metadata_translator/file_helpers.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Support functions for script implementations."""
14__all__ = ("find_files", "read_basic_metadata_from_file", "read_file_info")
16import json
17import re
18import os
19import sys
20import traceback
22from astro_metadata_translator import merge_headers, ObservationInfo
23from .tests import read_test_file
26# Prefer afw over Astropy
27try:
28 from lsst.afw.fits import readMetadata
29 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata
31 def _read_fits_metadata(file, hdu, can_raise=False):
32 """Read a FITS header using afw.
34 Parameters
35 ----------
36 file : `str`
37 The file to read.
38 hdu : `int`
39 The header number to read.
40 can_raise : `bool`, optional
41 Indicate whether the function can raise and exception (default)
42 or should return `None` on error. Can still raise if an unexpected
43 error is encountered.
45 Returns
46 -------
47 md : `dict`
48 The requested header. `None` if it could not be read and
49 ``can_raise`` is `False`.
51 Notes
52 -----
53 Tries to catch a FitsError 104 and convert to `FileNotFoundError`.
54 """
55 try:
56 return readMetadata(file, hdu=hdu)
57 except lsst.afw.fits.FitsError as e:
58 if can_raise:
59 # Try to convert a basic fits error code
60 if "(104)" in str(e):
61 raise FileNotFoundError(f"No such file or directory: {file}") from e
62 raise e
63 return None
65except ImportError:
66 from astropy.io import fits
68 def _read_fits_metadata(file, hdu, can_raise=False):
69 """Read a FITS header using astropy."""
71 # For detailed docstrings see the afw implementation above
72 header = None
73 try:
74 with fits.open(file) as fits_file:
75 try:
76 header = fits_file[hdu].header
77 except IndexError as e:
78 if can_raise:
79 raise e
80 except Exception as e:
81 if can_raise:
82 raise e
83 return header
86def find_files(files, regex):
87 """Find files for processing.
89 Parameters
90 ----------
91 files : iterable of `str`
92 The files or directories from which the headers are to be read.
93 regex : `str`
94 Regular expression string used to filter files when a directory is
95 scanned.
96 """
97 file_regex = re.compile(regex)
98 found_files = []
100 # Find all the files of interest
101 for file in files:
102 if os.path.isdir(file):
103 for root, dirs, files in os.walk(file):
104 for name in files:
105 path = os.path.join(root, name)
106 if os.path.isfile(path) and file_regex.search(name):
107 found_files.append(path)
108 else:
109 found_files.append(file)
111 return found_files
114def read_basic_metadata_from_file(file, hdrnum, errstream=sys.stderr, can_raise=False):
115 """Read a raw header from a file, merging if necessary
117 Parameters
118 ----------
119 file : `str`
120 Name of file to read. Can be FITS or YAML. YAML must be a simple
121 top-level dict.
122 hdrnum : `int`
123 Header number to read. Only relevant for FITS. If greater than 1
124 it will be merged with the primary header.
125 errstream : `io.StringIO`, optional
126 Stream to send messages that would normally be sent to standard
127 error. Defaults to `sys.stderr`. Only used if exceptions are disabled.
128 can_raise : `bool`, optional
129 Indicate whether the function can raise and exception (default)
130 or should return `None` on error. Can still raise if an unexpected
131 error is encountered.
133 Returns
134 -------
135 header : `dict`
136 The header as a dict. Can be `None` if there was a problem reading
137 the file.
138 """
139 if file.endswith(".yaml"):
140 try:
141 md = read_test_file(file,)
142 except Exception as e:
143 if not can_raise:
144 md = None
145 else:
146 raise e
147 if hdrnum != 0:
148 # YAML can't have HDUs so skip merging below
149 hdrnum = 0
150 else:
151 md = _read_fits_metadata(file, 0, can_raise=can_raise)
152 if md is None:
153 print(f"Unable to open file {file}", file=errstream)
154 return None
155 if hdrnum != 0:
156 mdn = _read_fits_metadata(file, int(hdrnum), can_raise=can_raise)
157 # Astropy does not allow append mode since it does not
158 # convert lists to multiple cards. Overwrite for now
159 if mdn is not None:
160 md = merge_headers([md, mdn], mode="overwrite")
161 else:
162 print(f"HDU {hdrnum} was not found in file {file}. Ignoring request.", file=errstream)
164 return md
167def read_file_info(file, hdrnum, print_trace=None, content_mode="translated", content_type="simple",
168 outstream=sys.stdout, errstream=sys.stderr):
169 """Read information from file
171 Parameters
172 ----------
173 file : `str`
174 The file from which the header is to be read.
175 hdrnum : `int`
176 The HDU number to read. The primary header is always read and
177 merged with the header from this HDU.
178 print_trace : `bool` or `None`
179 If there is an error reading the file and this parameter is `True`,
180 a full traceback of the exception will be reported. If `False` prints
181 a one line summary of the error condition. If `None` the exception
182 will be allowed to propagate.
183 content_mode : `str`
184 Content returned. This can be: ``metadata`` to return the unfixed
185 metadata headers; ``translated`` to return the output from metadata
186 translation.
187 content_type : `str`, optional
188 Form of content to be returned. Can be either ``json`` to return a
189 JSON string, ``simple`` to always return a `dict`, or ``native`` to
190 return either a `dict` (for ``metadata``) or `ObservationInfo` for
191 ``translated``.
192 outstream : `io.StringIO`, optional
193 Output stream to use for standard messages. Defaults to `sys.stdout`.
194 errstream : `io.StringIO`, optional
195 Stream to send messages that would normally be sent to standard
196 error. Defaults to `sys.stderr`.
198 Returns
199 -------
200 simple : `dict` of `str` or `ObservationInfo`
201 The return value of `ObservationInfo.to_simple()`. Returns `None`
202 if there was a problem and `print_trace` is not `None`.
203 """
205 if content_mode not in ("metadata", "translated"):
206 raise ValueError(f"Unrecognized content mode request: {content_mode}")
208 if content_type not in ("native", "simple", "json"):
209 raise ValueError(f"Unrecognized content type request {content_type}")
211 try:
212 # Calculate the JSON from the file
213 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream,
214 can_raise=True if print_trace is None else False)
215 if md is None:
216 return None
217 if content_mode == "metadata":
218 # Do not fix the header
219 if content_type == "json":
220 # Add a key to tell the reader whether this is md or translated
221 md["__CONTENT__"] = content_mode
222 try:
223 json_str = json.dumps(md)
224 except TypeError:
225 # Cast to dict and try again -- PropertyList is a problem
226 json_str = json.dumps(dict(md))
227 return json_str
228 return md
229 obs_info = ObservationInfo(md, pedantic=True, filename=file)
230 if content_type == "native":
231 return obs_info
232 simple = obs_info.to_simple()
233 if content_type == "simple":
234 return simple
235 if content_type == "json":
236 # Add a key to tell the reader if this is metadata or translated
237 simple["__CONTENT__"] = content_mode
238 return json.dumps(simple)
239 raise RuntimeError(f"Logic error. Unrecognized mode for reading file: {content_mode}/{content_type}")
240 except Exception as e:
241 if print_trace is None:
242 raise e
243 if print_trace:
244 traceback.print_exc(file=outstream)
245 else:
246 print(repr(e), file=outstream)
247 return None