Coverage for python/astro_metadata_translator/bin/writeindex.py: 21%

37 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-06 03:48 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ["write_index_files"] 

15 

16import json 

17import logging 

18import os 

19from collections.abc import MutableMapping, Sequence 

20from typing import IO 

21 

22from ..file_helpers import find_files 

23from ..indexing import index_files 

24 

25log = logging.getLogger(__name__) 

26 

27 

28def write_index_files( 

29 files: Sequence[str], 

30 regex: str, 

31 hdrnum: int, 

32 print_trace: bool, 

33 content_mode: str = "translated", 

34 outpath: str | None = None, 

35 outstream: IO | None = None, 

36) -> tuple[list[str], list[str]]: 

37 """Process each file and create JSON index file. 

38 

39 The index file will have common information in the toplevel. 

40 There is then a ``__DIFF__`` key that is a dictionary with file 

41 names as keys and per-file differences as the values in a dict. 

42 

43 Parameters 

44 ---------- 

45 files : iterable of `str` 

46 The files or directories from which the headers are to be read. 

47 regex : `str` 

48 Regular expression string used to filter files when a directory is 

49 scanned. 

50 hdrnum : `int` 

51 The HDU number to read. The primary header is always read and merged 

52 with the specified header. 

53 print_trace : `bool` 

54 If there is an error reading the file and this parameter is `True`, 

55 a full traceback of the exception will be reported. If `False` prints 

56 a one line summary of the error condition. 

57 content_mode : `str` 

58 Form of data to write in index file. Options are: 

59 ``translated`` (default) to write ObservationInfo to the index; 

60 ``metadata`` to write native metadata headers to the index. 

61 The index file is called ``_index.json``. 

62 outpath : `str`, optional 

63 If specified a single index file will be written to this location 

64 combining all the information from all files. If `None`, the default, 

65 and index file will be written to each directory in which files 

66 are found. 

67 outstream : `io.StringIO`, optional 

68 Output stream to use for standard messages. Defaults to `None` which 

69 uses the default output stream. Defaults to `sys.stdout`. 

70 

71 Returns 

72 ------- 

73 okay : `list` of `str` 

74 All the files that were processed successfully. 

75 failed : `list` of `str` 

76 All the files that could not be processed. 

77 """ 

78 if content_mode not in ("translated", "metadata"): 

79 raise ValueError(f"Unrecognized content mode {content_mode}") 

80 

81 if outpath is not None: 

82 _, ext = os.path.splitext(outpath) 

83 if ext != ".json": 

84 raise ValueError(f"Override output file must end in .json but given {outpath}") 

85 

86 found_files = find_files(files, regex) 

87 

88 failed = [] 

89 okay = [] 

90 files_per_directory: MutableMapping[str, list[str]] = {} 

91 

92 # Group each file by directory if no explicit output path 

93 if outpath is None: 

94 for path in found_files: 

95 head, tail = os.path.split(path) 

96 files_per_directory.setdefault(head, []).append(tail) 

97 else: 

98 files_per_directory["."] = list(found_files) 

99 

100 # Extract translated metadata for each file in each directory 

101 for directory, files_in_dir in files_per_directory.items(): 

102 output, this_okay, this_failed = index_files( 

103 files_in_dir, 

104 directory, 

105 hdrnum, 

106 print_trace, 

107 content_mode, 

108 outstream, 

109 ) 

110 

111 failed.extend(this_failed) 

112 okay.extend(this_okay) 

113 

114 # Write the index file 

115 if outpath is None: 

116 index_file = os.path.join(directory, "_index.json") 

117 else: 

118 index_file = outpath 

119 with open(index_file, "w") as fd: 

120 print(json.dumps(output), file=fd) 

121 log.info("Wrote index file to %s", index_file) 

122 

123 return okay, failed