Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Support functions for script implementations.""" 

13 

14__all__ = ("find_files", "read_basic_metadata_from_file", "read_file_info") 

15 

16import json 

17import re 

18import os 

19import sys 

20import traceback 

21 

22from .headers import merge_headers 

23from .observationInfo import ObservationInfo 

24from .tests import read_test_file 

25 

26 

27# Prefer afw over Astropy 

28try: 

29 from lsst.afw.fits import readMetadata 

30 import lsst.daf.base # noqa: F401 need PropertyBase for readMetadata 

31 

32 def _read_fits_metadata(file, hdu, can_raise=False): 

33 """Read a FITS header using afw. 

34 

35 Parameters 

36 ---------- 

37 file : `str` 

38 The file to read. 

39 hdu : `int` 

40 The header number to read. 

41 can_raise : `bool`, optional 

42 Indicate whether the function can raise and exception (default) 

43 or should return `None` on error. Can still raise if an unexpected 

44 error is encountered. 

45 

46 Returns 

47 ------- 

48 md : `dict` 

49 The requested header. `None` if it could not be read and 

50 ``can_raise`` is `False`. 

51 

52 Notes 

53 ----- 

54 Tries to catch a FitsError 104 and convert to `FileNotFoundError`. 

55 """ 

56 try: 

57 return readMetadata(file, hdu=hdu) 

58 except lsst.afw.fits.FitsError as e: 

59 if can_raise: 

60 # Try to convert a basic fits error code 

61 if "(104)" in str(e): 

62 raise FileNotFoundError(f"No such file or directory: {file}") from e 

63 raise e 

64 return None 

65 

66except ImportError: 

67 from astropy.io import fits 

68 

69 def _read_fits_metadata(file, hdu, can_raise=False): 

70 """Read a FITS header using astropy.""" 

71 

72 # For detailed docstrings see the afw implementation above 

73 header = None 

74 try: 

75 with fits.open(file) as fits_file: 

76 try: 

77 header = fits_file[hdu].header 

78 except IndexError as e: 

79 if can_raise: 

80 raise e 

81 except Exception as e: 

82 if can_raise: 

83 raise e 

84 return header 

85 

86 

87def find_files(files, regex): 

88 """Find files for processing. 

89 

90 Parameters 

91 ---------- 

92 files : iterable of `str` 

93 The files or directories from which the headers are to be read. 

94 regex : `str` 

95 Regular expression string used to filter files when a directory is 

96 scanned. 

97 """ 

98 file_regex = re.compile(regex) 

99 found_files = [] 

100 

101 # Find all the files of interest 

102 for file in files: 

103 if os.path.isdir(file): 

104 for root, dirs, files in os.walk(file): 

105 for name in files: 

106 path = os.path.join(root, name) 

107 if os.path.isfile(path) and file_regex.search(name): 

108 found_files.append(path) 

109 else: 

110 found_files.append(file) 

111 

112 return found_files 

113 

114 

115def read_basic_metadata_from_file(file, hdrnum, errstream=sys.stderr, can_raise=True): 

116 """Read a raw header from a file, merging if necessary 

117 

118 Parameters 

119 ---------- 

120 file : `str` 

121 Name of file to read. Can be FITS or YAML. YAML must be a simple 

122 top-level dict. 

123 hdrnum : `int` 

124 Header number to read. Only relevant for FITS. If greater than 1 

125 it will be merged with the primary header. If a negative number is 

126 given the second header, if present, will be merged with the primary 

127 header. If there is only a primary header a negative number behaves 

128 identically to specifying 0 for the HDU number. 

129 errstream : `io.StringIO`, optional 

130 Stream to send messages that would normally be sent to standard 

131 error. Defaults to `sys.stderr`. Only used if exceptions are disabled. 

132 can_raise : `bool`, optional 

133 Indicate whether the function can raise an exception (default) 

134 or should return `None` on error. Can still raise if an unexpected 

135 error is encountered. 

136 

137 Returns 

138 ------- 

139 header : `dict` 

140 The header as a dict. Can be `None` if there was a problem reading 

141 the file. 

142 """ 

143 if file.endswith(".yaml"): 

144 try: 

145 md = read_test_file(file,) 

146 except Exception as e: 

147 if not can_raise: 

148 md = None 

149 else: 

150 raise e 

151 if hdrnum != 0: 

152 # YAML can't have HDUs so skip merging below 

153 hdrnum = 0 

154 else: 

155 md = _read_fits_metadata(file, 0, can_raise=can_raise) 

156 if md is None: 

157 print(f"Unable to open file {file}", file=errstream) 

158 return None 

159 if hdrnum < 0: 

160 if "EXTEND" in md and md["EXTEND"]: 

161 hdrnum = 1 

162 if hdrnum > 0: 

163 # Allow this to fail 

164 mdn = _read_fits_metadata(file, int(hdrnum), can_raise=False) 

165 # Astropy does not allow append mode since it does not 

166 # convert lists to multiple cards. Overwrite for now 

167 if mdn is not None: 

168 md = merge_headers([md, mdn], mode="overwrite") 

169 else: 

170 print(f"HDU {hdrnum} was not found in file {file}. Ignoring request.", file=errstream) 

171 

172 return md 

173 

174 

175def read_file_info(file, hdrnum, print_trace=None, content_mode="translated", content_type="simple", 

176 outstream=sys.stdout, errstream=sys.stderr): 

177 """Read information from file 

178 

179 Parameters 

180 ---------- 

181 file : `str` 

182 The file from which the header is to be read. 

183 hdrnum : `int` 

184 The HDU number to read. The primary header is always read and 

185 merged with the header from this HDU. 

186 print_trace : `bool` or `None` 

187 If there is an error reading the file and this parameter is `True`, 

188 a full traceback of the exception will be reported. If `False` prints 

189 a one line summary of the error condition. If `None` the exception 

190 will be allowed to propagate. 

191 content_mode : `str` 

192 Content returned. This can be: ``metadata`` to return the unfixed 

193 metadata headers; ``translated`` to return the output from metadata 

194 translation. 

195 content_type : `str`, optional 

196 Form of content to be returned. Can be either ``json`` to return a 

197 JSON string, ``simple`` to always return a `dict`, or ``native`` to 

198 return either a `dict` (for ``metadata``) or `ObservationInfo` for 

199 ``translated``. 

200 outstream : `io.StringIO`, optional 

201 Output stream to use for standard messages. Defaults to `sys.stdout`. 

202 errstream : `io.StringIO`, optional 

203 Stream to send messages that would normally be sent to standard 

204 error. Defaults to `sys.stderr`. 

205 

206 Returns 

207 ------- 

208 simple : `dict` of `str` or `ObservationInfo` 

209 The return value of `ObservationInfo.to_simple()`. Returns `None` 

210 if there was a problem and `print_trace` is not `None`. 

211 """ 

212 

213 if content_mode not in ("metadata", "translated"): 

214 raise ValueError(f"Unrecognized content mode request: {content_mode}") 

215 

216 if content_type not in ("native", "simple", "json"): 

217 raise ValueError(f"Unrecognized content type request {content_type}") 

218 

219 try: 

220 # Calculate the JSON from the file 

221 md = read_basic_metadata_from_file(file, hdrnum, errstream=errstream, 

222 can_raise=True if print_trace is None else False) 

223 if md is None: 

224 return None 

225 if content_mode == "metadata": 

226 # Do not fix the header 

227 if content_type == "json": 

228 # Add a key to tell the reader whether this is md or translated 

229 md["__CONTENT__"] = content_mode 

230 try: 

231 json_str = json.dumps(md) 

232 except TypeError: 

233 # Cast to dict and try again -- PropertyList is a problem 

234 json_str = json.dumps(dict(md)) 

235 return json_str 

236 return md 

237 obs_info = ObservationInfo(md, pedantic=True, filename=file) 

238 if content_type == "native": 

239 return obs_info 

240 simple = obs_info.to_simple() 

241 if content_type == "simple": 

242 return simple 

243 if content_type == "json": 

244 # Add a key to tell the reader if this is metadata or translated 

245 simple["__CONTENT__"] = content_mode 

246 return json.dumps(simple) 

247 raise RuntimeError(f"Logic error. Unrecognized mode for reading file: {content_mode}/{content_type}") 

248 except Exception as e: 

249 if print_trace is None: 

250 raise e 

251 if print_trace: 

252 traceback.print_exc(file=outstream) 

253 else: 

254 print(repr(e), file=outstream) 

255 return None