Coverage for python/felis/cli.py: 61%
131 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:49 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:49 -0700
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from __future__ import annotations
24import io
25import logging
26from collections.abc import Iterable
27from typing import IO
29import click
30import yaml
31from pydantic import ValidationError
32from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
33from sqlalchemy.engine.mock import MockConnection
35from . import __version__
36from .datamodel import Schema
37from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
38from .tap import Tap11Base, TapLoadingVisitor, init_tables
39from .validation import get_schema
41logger = logging.getLogger("felis")
43loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
46@click.group()
47@click.version_option(__version__)
48@click.option(
49 "--log-level",
50 type=click.Choice(loglevel_choices),
51 envvar="FELIS_LOGLEVEL",
52 help="Felis log level",
53 default=logging.getLevelName(logging.INFO),
54)
55@click.option(
56 "--log-file",
57 type=click.Path(),
58 envvar="FELIS_LOGFILE",
59 help="Felis log file path",
60)
61def cli(log_level: str, log_file: str | None) -> None:
62 """Felis Command Line Tools."""
63 if log_file:
64 logging.basicConfig(filename=log_file, level=log_level)
65 else:
66 logging.basicConfig(level=log_level)
69@cli.command("create")
70@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
71@click.option("--schema-name", help="Alternate schema name to override Felis file")
72@click.option(
73 "--create-if-not-exists", is_flag=True, help="Create the schema in the database if it does not exist"
74)
75@click.option("--drop-if-exists", is_flag=True, help="Drop schema if it already exists in the database")
76@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
77@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
78@click.option(
79 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
80)
81@click.argument("file", type=click.File())
82def create(
83 engine_url: str,
84 schema_name: str | None,
85 create_if_not_exists: bool,
86 drop_if_exists: bool,
87 echo: bool,
88 dry_run: bool,
89 output_file: IO[str] | None,
90 file: IO,
91) -> None:
92 """Create database objects from the Felis file."""
93 yaml_data = yaml.safe_load(file)
94 schema = Schema.model_validate(yaml_data)
95 url_obj = make_url(engine_url)
96 if schema_name:
97 logger.info(f"Overriding schema name with: {schema_name}")
98 schema.name = schema_name
99 elif url_obj.drivername == "sqlite":
100 logger.info("Overriding schema name for sqlite with: main")
101 schema.name = "main"
102 if not url_obj.host and not url_obj.drivername == "sqlite":
103 dry_run = True
104 logger.info("Forcing dry run for non-sqlite engine URL with no host")
106 builder = MetaDataBuilder(schema)
107 builder.build()
108 metadata = builder.metadata
109 logger.debug(f"Created metadata with schema name: {metadata.schema}")
111 engine: Engine | MockConnection
112 if not dry_run and not output_file:
113 engine = create_engine(engine_url, echo=echo)
114 else:
115 if dry_run:
116 logger.info("Dry run will be executed")
117 engine = DatabaseContext.create_mock_engine(url_obj, output_file)
118 if output_file:
119 logger.info("Writing SQL output to: " + output_file.name)
121 context = DatabaseContext(metadata, engine)
123 if drop_if_exists:
124 logger.debug("Dropping schema if it exists")
125 context.drop_if_exists()
126 create_if_not_exists = True # If schema is dropped, it needs to be recreated.
128 if create_if_not_exists:
129 logger.debug("Creating schema if not exists")
130 context.create_if_not_exists()
132 context.create_all()
135@cli.command("init-tap")
136@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
137@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
138@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
139@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
140@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
141@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
142@click.argument("engine-url")
143def init_tap(
144 engine_url: str,
145 tap_schema_name: str,
146 tap_schemas_table: str,
147 tap_tables_table: str,
148 tap_columns_table: str,
149 tap_keys_table: str,
150 tap_key_columns_table: str,
151) -> None:
152 """Initialize TAP 1.1 TAP_SCHEMA objects.
154 Please verify the schema/catalog you are executing this in in your
155 engine URL.
156 """
157 engine = create_engine(engine_url, echo=True)
158 init_tables(
159 tap_schema_name,
160 tap_schemas_table,
161 tap_tables_table,
162 tap_columns_table,
163 tap_keys_table,
164 tap_key_columns_table,
165 )
166 Tap11Base.metadata.create_all(engine)
169@cli.command("load-tap")
170@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
171@click.option("--schema-name", help="Alternate Schema Name for Felis file")
172@click.option("--catalog-name", help="Catalog Name for Schema")
173@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
174@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
175@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
176@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
177@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
178@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
179@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
180@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
181@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema")
182@click.argument("file", type=click.File())
183def load_tap(
184 engine_url: str,
185 schema_name: str,
186 catalog_name: str,
187 dry_run: bool,
188 tap_schema_name: str,
189 tap_tables_postfix: str,
190 tap_schemas_table: str,
191 tap_tables_table: str,
192 tap_columns_table: str,
193 tap_keys_table: str,
194 tap_key_columns_table: str,
195 tap_schema_index: int,
196 file: io.TextIOBase,
197) -> None:
198 """Load TAP metadata from a Felis FILE.
200 This command loads the associated TAP metadata from a Felis FILE
201 to the TAP_SCHEMA tables.
202 """
203 yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
204 schema = Schema.model_validate(yaml_data)
206 tap_tables = init_tables(
207 tap_schema_name,
208 tap_tables_postfix,
209 tap_schemas_table,
210 tap_tables_table,
211 tap_columns_table,
212 tap_keys_table,
213 tap_key_columns_table,
214 )
216 if not dry_run:
217 engine = create_engine(engine_url)
219 if engine_url == "sqlite://" and not dry_run:
220 # In Memory SQLite - Mostly used to test
221 Tap11Base.metadata.create_all(engine)
223 tap_visitor = TapLoadingVisitor(
224 engine,
225 catalog_name=catalog_name,
226 schema_name=schema_name,
227 tap_tables=tap_tables,
228 tap_schema_index=tap_schema_index,
229 )
230 tap_visitor.visit_schema(schema)
231 else:
232 _insert_dump = InsertDump()
233 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
234 # After the engine is created, update the executor with the dialect
235 _insert_dump.dialect = conn.dialect
237 tap_visitor = TapLoadingVisitor.from_mock_connection(
238 conn,
239 catalog_name=catalog_name,
240 schema_name=schema_name,
241 tap_tables=tap_tables,
242 tap_schema_index=tap_schema_index,
243 )
244 tap_visitor.visit_schema(schema)
247@cli.command("validate")
248@click.option(
249 "-s",
250 "--schema-name",
251 help="Schema name for validation",
252 type=click.Choice(["RSP", "default"]),
253 default="default",
254)
255@click.option(
256 "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
257)
258@click.option(
259 "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
260)
261@click.argument("files", nargs=-1, type=click.File())
262def validate(
263 schema_name: str,
264 require_description: bool,
265 check_redundant_datatypes: bool,
266 files: Iterable[io.TextIOBase],
267) -> None:
268 """Validate one or more felis YAML files."""
269 schema_class = get_schema(schema_name)
270 if schema_name != "default":
271 logger.info(f"Using schema '{schema_class.__name__}'")
273 rc = 0
274 for file in files:
275 file_name = getattr(file, "name", None)
276 logger.info(f"Validating {file_name}")
277 try:
278 data = yaml.load(file, Loader=yaml.SafeLoader)
279 schema_class.model_validate(
280 data,
281 context={
282 "check_redundant_datatypes": check_redundant_datatypes,
283 "require_description": require_description,
284 },
285 )
286 except ValidationError as e:
287 logger.error(e)
288 rc = 1
289 if rc:
290 raise click.exceptions.Exit(rc)
293if __name__ == "__main__": 293 ↛ 294line 293 didn't jump to line 294, because the condition on line 293 was never true
294 cli()