Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Support functions for script implementations.""" 

13 

14__all__ = ("find_files", "read_basic_metadata_from_file", "read_file_info") 

15 

16import json 

17import re 

18import os 

19import sys 

20import traceback 

21 

22from astro_metadata_translator import merge_headers, ObservationInfo 

23from .tests import read_test_file 

24 

25 

26# Prefer afw over Astropy 

27try: 

28 from lsst.afw.fits import readMetadata 

29 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata 

30 

31 def _read_fits_metadata(file, hdu, can_raise=False): 

32 """Read a FITS header using afw. 

33 

34 Parameters 

35 ---------- 

36 file : `str` 

37 The file to read. 

38 hdu : `int` 

39 The header number to read. 

40 can_raise : `bool`, optional 

41 Indicate whether the function can raise and exception (default) 

42 or should return `None` on error. Can still raise if an unexpected 

43 error is encountered. 

44 

45 Returns 

46 ------- 

47 md : `dict` 

48 The requested header. `None` if it could not be read and 

49 ``can_raise`` is `False`. 

50 

51 Notes 

52 ----- 

53 Tries to catch a FitsError 104 and convert to `FileNotFoundError`. 

54 """ 

55 try: 

56 return readMetadata(file, hdu=hdu) 

57 except lsst.afw.fits.FitsError as e: 

58 if can_raise: 

59 # Try to convert a basic fits error code 

60 if "(104)" in str(e): 

61 raise FileNotFoundError(f"No such file or directory: {file}") from e 

62 raise e 

63 return None 

64 

65except ImportError: 

66 from astropy.io import fits 

67 

68 def _read_fits_metadata(file, hdu, can_raise=False): 

69 """Read a FITS header using astropy.""" 

70 

71 # For detailed docstrings see the afw implementation above 

72 header = None 

73 try: 

74 with fits.open(file) as fits_file: 

75 try: 

76 header = fits_file[hdu].header 

77 except IndexError as e: 

78 if can_raise: 

79 raise e 

80 except Exception as e: 

81 if can_raise: 

82 raise e 

83 return header 

84 

85 

86def find_files(files, regex): 

87 """Find files for processing. 

88 

89 Parameters 

90 ---------- 

91 files : iterable of `str` 

92 The files or directories from which the headers are to be read. 

93 regex : `str` 

94 Regular expression string used to filter files when a directory is 

95 scanned. 

96 """ 

97 file_regex = re.compile(regex) 

98 found_files = [] 

99 

100 # Find all the files of interest 

101 for file in files: 

102 if os.path.isdir(file): 

103 for root, dirs, files in os.walk(file): 

104 for name in files: 

105 path = os.path.join(root, name) 

106 if os.path.isfile(path) and file_regex.search(name): 

107 found_files.append(path) 

108 else: 

109 found_files.append(file) 

110 

111 return found_files 

112 

113 

114def read_basic_metadata_from_file(file, hdrnum, errstream=sys.stderr, can_raise=False): 

115 """Read a raw header from a file, merging if necessary 

116 

117 Parameters 

118 ---------- 

119 file : `str` 

120 Name of file to read. Can be FITS or YAML. YAML must be a simple 

121 top-level dict. 

122 hdrnum : `int` 

123 Header number to read. Only relevant for FITS. If greater than 1 

124 it will be merged with the primary header. 

125 errstream : `io.StringIO`, optional 

126 Stream to send messages that would normally be sent to standard 

127 error. Defaults to `sys.stderr`. Only used if exceptions are disabled. 

128 can_raise : `bool`, optional 

129 Indicate whether the function can raise and exception (default) 

130 or should return `None` on error. Can still raise if an unexpected 

131 error is encountered. 

132 

133 Returns 

134 ------- 

135 header : `dict` 

136 The header as a dict. Can be `None` if there was a problem reading 

137 the file. 

138 """ 

139 if file.endswith(".yaml"): 

140 try: 

141 md = read_test_file(file,) 

142 except Exception as e: 

143 if not can_raise: 

144 md = None 

145 else: 

146 raise e 

147 if hdrnum != 0: 

148 # YAML can't have HDUs so skip merging below 

149 hdrnum = 0 

150 else: 

151 md = _read_fits_metadata(file, 0, can_raise=can_raise) 

152 if md is None: 

153 print(f"Unable to open file {file}", file=errstream) 

154 return None 

155 if hdrnum != 0: 

156 mdn = _read_fits_metadata(file, int(hdrnum), can_raise=can_raise) 

157 # Astropy does not allow append mode since it does not 

158 # convert lists to multiple cards. Overwrite for now 

159 if mdn is not None: 

160 md = merge_headers([md, mdn], mode="overwrite") 

161 else: 

162 print(f"HDU {hdrnum} was not found in file {file}. Ignoring request.", file=errstream) 

163 

164 return md 

165 

166 

167def read_file_info(file, hdrnum, print_trace=None, content_mode="translated", content_type="simple", 

168 outstream=sys.stdout, errstream=sys.stderr): 

169 """Read information from file 

170 

171 Parameters 

172 ---------- 

173 file : `str` 

174 The file from which the header is to be read. 

175 hdrnum : `int` 

176 The HDU number to read. The primary header is always read and 

177 merged with the header from this HDU. 

178 print_trace : `bool` or `None` 

179 If there is an error reading the file and this parameter is `True`, 

180 a full traceback of the exception will be reported. If `False` prints 

181 a one line summary of the error condition. If `None` the exception 

182 will be allowed to propagate. 

183 content_mode : `str` 

184 Content returned. This can be: ``metadata`` to return the unfixed 

185 metadata headers; ``translated`` to return the output from metadata 

186 translation. 

187 content_type : `str`, optional 

188 Form of content to be returned. Can be either ``json`` to return a 

189 JSON string, ``simple`` to always return a `dict`, or ``native`` to 

190 return either a `dict` (for ``metadata``) or `ObservationInfo` for 

191 ``translated``. 

192 outstream : `io.StringIO`, optional 

193 Output stream to use for standard messages. Defaults to `sys.stdout`. 

194 errstream : `io.StringIO`, optional 

195 Stream to send messages that would normally be sent to standard 

196 error. Defaults to `sys.stderr`. 

197 

198 Returns 

199 ------- 

200 simple : `dict` of `str` or `ObservationInfo` 

201 The return value of `ObservationInfo.to_simple()`. Returns `None` 

202 if there was a problem and `print_trace` is not `None`. 

203 """ 

204 

205 if content_mode not in ("metadata", "translated"): 

206 raise ValueError(f"Unrecognized content mode request: {content_mode}") 

207 

208 if content_type not in ("native", "simple", "json"): 

209 raise ValueError(f"Unrecognized content type request {content_type}") 

210 

211 try: 

212 # Calculate the JSON from the file 

213 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, 

214 can_raise=True if print_trace is None else False) 

215 if md is None: 

216 return None 

217 if content_mode == "metadata": 

218 # Do not fix the header 

219 if content_type == "json": 

220 # Add a key to tell the reader whether this is md or translated 

221 md["__CONTENT__"] = content_mode 

222 try: 

223 json_str = json.dumps(md) 

224 except TypeError: 

225 # Cast to dict and try again -- PropertyList is a problem 

226 json_str = json.dumps(dict(md)) 

227 return json_str 

228 return md 

229 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

230 if content_type == "native": 

231 return obs_info 

232 simple = obs_info.to_simple() 

233 if content_type == "simple": 

234 return simple 

235 if content_type == "json": 

236 # Add a key to tell the reader if this is metadata or translated 

237 simple["__CONTENT__"] = content_mode 

238 return json.dumps(simple) 

239 raise RuntimeError(f"Logic error. Unrecognized mode for reading file: {content_mode}/{content_type}") 

240 except Exception as e: 

241 if print_trace is None: 

242 raise e 

243 if print_trace: 

244 traceback.print_exc(file=outstream) 

245 else: 

246 print(repr(e), file=outstream) 

247 return None