Coverage for python/astro_metadata_translator/file_helpers.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Support functions for script implementations."""
14__all__ = ("find_files", "read_basic_metadata_from_file", "read_file_info")
16import json
17import re
18import os
19import sys
20import traceback
22from .headers import merge_headers
23from .observationInfo import ObservationInfo
24from .tests import read_test_file
27# Prefer afw over Astropy
28try:
29 from lsst.afw.fits import readMetadata
30 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata
32 def _read_fits_metadata(file, hdu, can_raise=False):
33 """Read a FITS header using afw.
35 Parameters
36 ----------
37 file : `str`
38 The file to read.
39 hdu : `int`
40 The header number to read.
41 can_raise : `bool`, optional
42 Indicate whether the function can raise and exception (default)
43 or should return `None` on error. Can still raise if an unexpected
44 error is encountered.
46 Returns
47 -------
48 md : `dict`
49 The requested header. `None` if it could not be read and
50 ``can_raise`` is `False`.
52 Notes
53 -----
54 Tries to catch a FitsError 104 and convert to `FileNotFoundError`.
55 """
56 try:
57 return readMetadata(file, hdu=hdu)
58 except lsst.afw.fits.FitsError as e:
59 if can_raise:
60 # Try to convert a basic fits error code
61 if "(104)" in str(e):
62 raise FileNotFoundError(f"No such file or directory: {file}") from e
63 raise e
64 return None
66except ImportError:
67 from astropy.io import fits
69 def _read_fits_metadata(file, hdu, can_raise=False):
70 """Read a FITS header using astropy."""
72 # For detailed docstrings see the afw implementation above
73 header = None
74 try:
75 with fits.open(file) as fits_file:
76 try:
77 header = fits_file[hdu].header
78 except IndexError as e:
79 if can_raise:
80 raise e
81 except Exception as e:
82 if can_raise:
83 raise e
84 return header
87def find_files(files, regex):
88 """Find files for processing.
90 Parameters
91 ----------
92 files : iterable of `str`
93 The files or directories from which the headers are to be read.
94 regex : `str`
95 Regular expression string used to filter files when a directory is
96 scanned.
97 """
98 file_regex = re.compile(regex)
99 found_files = []
101 # Find all the files of interest
102 for file in files:
103 if os.path.isdir(file):
104 for root, dirs, files in os.walk(file):
105 for name in files:
106 path = os.path.join(root, name)
107 if os.path.isfile(path) and file_regex.search(name):
108 found_files.append(path)
109 else:
110 found_files.append(file)
112 return found_files
115def read_basic_metadata_from_file(file, hdrnum, errstream=sys.stderr, can_raise=True):
116 """Read a raw header from a file, merging if necessary
118 Parameters
119 ----------
120 file : `str`
121 Name of file to read. Can be FITS or YAML. YAML must be a simple
122 top-level dict.
123 hdrnum : `int`
124 Header number to read. Only relevant for FITS. If greater than 1
125 it will be merged with the primary header. If a negative number is
126 given the second header, if present, will be merged with the primary
127 header. If there is only a primary header a negative number behaves
128 identically to specifying 0 for the HDU number.
129 errstream : `io.StringIO`, optional
130 Stream to send messages that would normally be sent to standard
131 error. Defaults to `sys.stderr`. Only used if exceptions are disabled.
132 can_raise : `bool`, optional
133 Indicate whether the function can raise an exception (default)
134 or should return `None` on error. Can still raise if an unexpected
135 error is encountered.
137 Returns
138 -------
139 header : `dict`
140 The header as a dict. Can be `None` if there was a problem reading
141 the file.
142 """
143 if file.endswith(".yaml"):
144 try:
145 md = read_test_file(file,)
146 except Exception as e:
147 if not can_raise:
148 md = None
149 else:
150 raise e
151 if hdrnum != 0:
152 # YAML can't have HDUs so skip merging below
153 hdrnum = 0
154 else:
155 md = _read_fits_metadata(file, 0, can_raise=can_raise)
156 if md is None:
157 print(f"Unable to open file {file}", file=errstream)
158 return None
159 if hdrnum < 0:
160 if "EXTEND" in md and md["EXTEND"]:
161 hdrnum = 1
162 if hdrnum > 0:
163 # Allow this to fail
164 mdn = _read_fits_metadata(file, int(hdrnum), can_raise=False)
165 # Astropy does not allow append mode since it does not
166 # convert lists to multiple cards. Overwrite for now
167 if mdn is not None:
168 md = merge_headers([md, mdn], mode="overwrite")
169 else:
170 print(f"HDU {hdrnum} was not found in file {file}. Ignoring request.", file=errstream)
172 return md
175def read_file_info(file, hdrnum, print_trace=None, content_mode="translated", content_type="simple",
176 outstream=sys.stdout, errstream=sys.stderr):
177 """Read information from file
179 Parameters
180 ----------
181 file : `str`
182 The file from which the header is to be read.
183 hdrnum : `int`
184 The HDU number to read. The primary header is always read and
185 merged with the header from this HDU.
186 print_trace : `bool` or `None`
187 If there is an error reading the file and this parameter is `True`,
188 a full traceback of the exception will be reported. If `False` prints
189 a one line summary of the error condition. If `None` the exception
190 will be allowed to propagate.
191 content_mode : `str`
192 Content returned. This can be: ``metadata`` to return the unfixed
193 metadata headers; ``translated`` to return the output from metadata
194 translation.
195 content_type : `str`, optional
196 Form of content to be returned. Can be either ``json`` to return a
197 JSON string, ``simple`` to always return a `dict`, or ``native`` to
198 return either a `dict` (for ``metadata``) or `ObservationInfo` for
199 ``translated``.
200 outstream : `io.StringIO`, optional
201 Output stream to use for standard messages. Defaults to `sys.stdout`.
202 errstream : `io.StringIO`, optional
203 Stream to send messages that would normally be sent to standard
204 error. Defaults to `sys.stderr`.
206 Returns
207 -------
208 simple : `dict` of `str` or `ObservationInfo`
209 The return value of `ObservationInfo.to_simple()`. Returns `None`
210 if there was a problem and `print_trace` is not `None`.
211 """
213 if content_mode not in ("metadata", "translated"):
214 raise ValueError(f"Unrecognized content mode request: {content_mode}")
216 if content_type not in ("native", "simple", "json"):
217 raise ValueError(f"Unrecognized content type request {content_type}")
219 try:
220 # Calculate the JSON from the file
221 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream,
222 can_raise=True if print_trace is None else False)
223 if md is None:
224 return None
225 if content_mode == "metadata":
226 # Do not fix the header
227 if content_type == "json":
228 # Add a key to tell the reader whether this is md or translated
229 md["__CONTENT__"] = content_mode
230 try:
231 json_str = json.dumps(md)
232 except TypeError:
233 # Cast to dict and try again -- PropertyList is a problem
234 json_str = json.dumps(dict(md))
235 return json_str
236 return md
237 obs_info = ObservationInfo(md, pedantic=True, filename=file)
238 if content_type == "native":
239 return obs_info
240 simple = obs_info.to_simple()
241 if content_type == "simple":
242 return simple
243 if content_type == "json":
244 # Add a key to tell the reader if this is metadata or translated
245 simple["__CONTENT__"] = content_mode
246 return json.dumps(simple)
247 raise RuntimeError(f"Logic error. Unrecognized mode for reading file: {content_mode}/{content_type}")
248 except Exception as e:
249 if print_trace is None:
250 raise e
251 if print_trace:
252 traceback.print_exc(file=outstream)
253 else:
254 print(repr(e), file=outstream)
255 return None