Coverage for python/astro_metadata_translator/bin/writeindex.py: 21%
37 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-06 03:48 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-06 03:48 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = ["write_index_files"]
16import json
17import logging
18import os
19from collections.abc import MutableMapping, Sequence
20from typing import IO
22from ..file_helpers import find_files
23from ..indexing import index_files
25log = logging.getLogger(__name__)
28def write_index_files(
29 files: Sequence[str],
30 regex: str,
31 hdrnum: int,
32 print_trace: bool,
33 content_mode: str = "translated",
34 outpath: str | None = None,
35 outstream: IO | None = None,
36) -> tuple[list[str], list[str]]:
37 """Process each file and create JSON index file.
39 The index file will have common information in the toplevel.
40 There is then a ``__DIFF__`` key that is a dictionary with file
41 names as keys and per-file differences as the values in a dict.
43 Parameters
44 ----------
45 files : iterable of `str`
46 The files or directories from which the headers are to be read.
47 regex : `str`
48 Regular expression string used to filter files when a directory is
49 scanned.
50 hdrnum : `int`
51 The HDU number to read. The primary header is always read and merged
52 with the specified header.
53 print_trace : `bool`
54 If there is an error reading the file and this parameter is `True`,
55 a full traceback of the exception will be reported. If `False` prints
56 a one line summary of the error condition.
57 content_mode : `str`
58 Form of data to write in index file. Options are:
59 ``translated`` (default) to write ObservationInfo to the index;
60 ``metadata`` to write native metadata headers to the index.
61 The index file is called ``_index.json``.
62 outpath : `str`, optional
63 If specified a single index file will be written to this location
64 combining all the information from all files. If `None`, the default,
65 and index file will be written to each directory in which files
66 are found.
67 outstream : `io.StringIO`, optional
68 Output stream to use for standard messages. Defaults to `None` which
69 uses the default output stream. Defaults to `sys.stdout`.
71 Returns
72 -------
73 okay : `list` of `str`
74 All the files that were processed successfully.
75 failed : `list` of `str`
76 All the files that could not be processed.
77 """
78 if content_mode not in ("translated", "metadata"):
79 raise ValueError(f"Unrecognized content mode {content_mode}")
81 if outpath is not None:
82 _, ext = os.path.splitext(outpath)
83 if ext != ".json":
84 raise ValueError(f"Override output file must end in .json but given {outpath}")
86 found_files = find_files(files, regex)
88 failed = []
89 okay = []
90 files_per_directory: MutableMapping[str, list[str]] = {}
92 # Group each file by directory if no explicit output path
93 if outpath is None:
94 for path in found_files:
95 head, tail = os.path.split(path)
96 files_per_directory.setdefault(head, []).append(tail)
97 else:
98 files_per_directory["."] = list(found_files)
100 # Extract translated metadata for each file in each directory
101 for directory, files_in_dir in files_per_directory.items():
102 output, this_okay, this_failed = index_files(
103 files_in_dir,
104 directory,
105 hdrnum,
106 print_trace,
107 content_mode,
108 outstream,
109 )
111 failed.extend(this_failed)
112 okay.extend(this_okay)
114 # Write the index file
115 if outpath is None:
116 index_file = os.path.join(directory, "_index.json")
117 else:
118 index_file = outpath
119 with open(index_file, "w") as fd:
120 print(json.dumps(output), file=fd)
121 log.info("Wrote index file to %s", index_file)
123 return okay, failed