Coverage for python/felis/cli.py: 54%
228 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-25 10:20 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-25 10:20 -0700
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from __future__ import annotations
24import io
25import json
26import logging
27import sys
28from collections.abc import Iterable, Mapping, MutableMapping
29from typing import IO, Any
31import click
32import yaml
33from pydantic import ValidationError
34from pyld import jsonld
35from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
36from sqlalchemy.engine.mock import MockConnection
38from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
39from .check import CheckingVisitor
40from .datamodel import Schema
41from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
42from .tap import Tap11Base, TapLoadingVisitor, init_tables
43from .utils import ReorderingVisitor
44from .validation import get_schema
46logger = logging.getLogger("felis")
48loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
51@click.group()
52@click.version_option(__version__)
53@click.option(
54 "--log-level",
55 type=click.Choice(loglevel_choices),
56 envvar="FELIS_LOGLEVEL",
57 help="Felis log level",
58 default=logging.getLevelName(logging.INFO),
59)
60@click.option(
61 "--log-file",
62 type=click.Path(),
63 envvar="FELIS_LOGFILE",
64 help="Felis log file path",
65)
66def cli(log_level: str, log_file: str | None) -> None:
67 """Felis Command Line Tools."""
68 if log_file:
69 logging.basicConfig(filename=log_file, level=log_level)
70 else:
71 logging.basicConfig(level=log_level)
74@cli.command("create")
75@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
76@click.option("--schema-name", help="Alternate schema name to override Felis file")
77@click.option(
78 "--create-if-not-exists", is_flag=True, help="Create the schema in the database if it does not exist"
79)
80@click.option("--drop-if-exists", is_flag=True, help="Drop schema if it already exists in the database")
81@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
82@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
83@click.option(
84 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
85)
86@click.argument("file", type=click.File())
87def create(
88 engine_url: str,
89 schema_name: str | None,
90 create_if_not_exists: bool,
91 drop_if_exists: bool,
92 echo: bool,
93 dry_run: bool,
94 output_file: IO[str] | None,
95 file: IO,
96) -> None:
97 """Create database objects from the Felis file."""
98 yaml_data = yaml.safe_load(file)
99 schema = Schema.model_validate(yaml_data)
100 url_obj = make_url(engine_url)
101 if schema_name:
102 logger.info(f"Overriding schema name with: {schema_name}")
103 schema.name = schema_name
104 elif url_obj.drivername == "sqlite":
105 logger.info("Overriding schema name for sqlite with: main")
106 schema.name = "main"
107 if not url_obj.host and not url_obj.drivername == "sqlite":
108 dry_run = True
109 logger.info("Forcing dry run for non-sqlite engine URL with no host")
111 builder = MetaDataBuilder(schema)
112 builder.build()
113 metadata = builder.metadata
114 logger.debug(f"Created metadata with schema name: {metadata.schema}")
116 engine: Engine | MockConnection
117 if not dry_run and not output_file:
118 engine = create_engine(engine_url, echo=echo)
119 else:
120 if dry_run:
121 logger.info("Dry run will be executed")
122 engine = DatabaseContext.create_mock_engine(url_obj, output_file)
123 if output_file:
124 logger.info("Writing SQL output to: " + output_file.name)
126 context = DatabaseContext(metadata, engine)
128 if drop_if_exists:
129 logger.debug("Dropping schema if it exists")
130 context.drop_if_exists()
131 create_if_not_exists = True # If schema is dropped, it needs to be recreated.
133 if create_if_not_exists:
134 logger.debug("Creating schema if not exists")
135 context.create_if_not_exists()
137 context.create_all()
140@cli.command("init-tap")
141@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
142@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
143@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
144@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
145@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
146@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
147@click.argument("engine-url")
148def init_tap(
149 engine_url: str,
150 tap_schema_name: str,
151 tap_schemas_table: str,
152 tap_tables_table: str,
153 tap_columns_table: str,
154 tap_keys_table: str,
155 tap_key_columns_table: str,
156) -> None:
157 """Initialize TAP 1.1 TAP_SCHEMA objects.
159 Please verify the schema/catalog you are executing this in in your
160 engine URL.
161 """
162 engine = create_engine(engine_url, echo=True)
163 init_tables(
164 tap_schema_name,
165 tap_schemas_table,
166 tap_tables_table,
167 tap_columns_table,
168 tap_keys_table,
169 tap_key_columns_table,
170 )
171 Tap11Base.metadata.create_all(engine)
174@cli.command("load-tap")
175@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
176@click.option("--schema-name", help="Alternate Schema Name for Felis file")
177@click.option("--catalog-name", help="Catalog Name for Schema")
178@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
179@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
180@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
181@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
182@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
183@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
184@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
185@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
186@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema")
187@click.argument("file", type=click.File())
188def load_tap(
189 engine_url: str,
190 schema_name: str,
191 catalog_name: str,
192 dry_run: bool,
193 tap_schema_name: str,
194 tap_tables_postfix: str,
195 tap_schemas_table: str,
196 tap_tables_table: str,
197 tap_columns_table: str,
198 tap_keys_table: str,
199 tap_key_columns_table: str,
200 tap_schema_index: int,
201 file: io.TextIOBase,
202) -> None:
203 """Load TAP metadata from a Felis FILE.
205 This command loads the associated TAP metadata from a Felis FILE
206 to the TAP_SCHEMA tables.
207 """
208 yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
209 schema = Schema.model_validate(yaml_data)
211 tap_tables = init_tables(
212 tap_schema_name,
213 tap_tables_postfix,
214 tap_schemas_table,
215 tap_tables_table,
216 tap_columns_table,
217 tap_keys_table,
218 tap_key_columns_table,
219 )
221 if not dry_run:
222 engine = create_engine(engine_url)
224 if engine_url == "sqlite://" and not dry_run:
225 # In Memory SQLite - Mostly used to test
226 Tap11Base.metadata.create_all(engine)
228 tap_visitor = TapLoadingVisitor(
229 engine,
230 catalog_name=catalog_name,
231 schema_name=schema_name,
232 tap_tables=tap_tables,
233 tap_schema_index=tap_schema_index,
234 )
235 tap_visitor.visit_schema(schema)
236 else:
237 _insert_dump = InsertDump()
238 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
239 # After the engine is created, update the executor with the dialect
240 _insert_dump.dialect = conn.dialect
242 tap_visitor = TapLoadingVisitor.from_mock_connection(
243 conn,
244 catalog_name=catalog_name,
245 schema_name=schema_name,
246 tap_tables=tap_tables,
247 tap_schema_index=tap_schema_index,
248 )
249 tap_visitor.visit_schema(schema)
252@cli.command("modify-tap")
253@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0)
254@click.argument("files", nargs=-1, type=click.File())
255def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
256 """Modify TAP information in Felis schema FILES.
258 This command has some utilities to aid in rewriting felis FILES
259 in specific ways. It will write out a merged version of these files.
260 """
261 count = 0
262 graph = []
263 for file in files:
264 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
265 if "@graph" not in schema_obj:
266 schema_obj["@type"] = "felis:Schema"
267 schema_obj["@context"] = DEFAULT_CONTEXT
268 schema_index = schema_obj.get("tap:schema_index")
269 if not schema_index or (schema_index and schema_index > start_schema_at):
270 schema_index = start_schema_at + count
271 count += 1
272 schema_obj["tap:schema_index"] = schema_index
273 graph.extend(jsonld.flatten(schema_obj))
274 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
275 normalized = _normalize(merged, embed="@always")
276 _dump(normalized)
279@cli.command("basic-check")
280@click.argument("file", type=click.File())
281def basic_check(file: io.TextIOBase) -> None:
282 """Perform a basic check on a felis FILE.
284 This performs a very check to ensure required fields are
285 populated and basic semantics are okay. It does not ensure semantics
286 are valid for other commands like create-all or load-tap.
287 """
288 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
289 schema_obj["@type"] = "felis:Schema"
290 # Force Context and Schema Type
291 schema_obj["@context"] = DEFAULT_CONTEXT
292 check_visitor = CheckingVisitor()
293 check_visitor.visit_schema(schema_obj)
296@cli.command("normalize")
297@click.argument("file", type=click.File())
298def normalize(file: io.TextIOBase) -> None:
299 """Normalize a Felis FILE.
301 Takes a felis schema FILE, expands it (resolving the full URLs),
302 then compacts it, and finally produces output in the canonical
303 format.
305 (This is most useful in some debugging scenarios)
307 See Also :
309 https://json-ld.org/spec/latest/json-ld/#expanded-document-form
310 https://json-ld.org/spec/latest/json-ld/#compacted-document-form
311 """
312 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
313 schema_obj["@type"] = "felis:Schema"
314 # Force Context and Schema Type
315 schema_obj["@context"] = DEFAULT_CONTEXT
316 expanded = jsonld.expand(schema_obj)
317 normalized = _normalize(expanded, embed="@always")
318 _dump(normalized)
321@cli.command("merge")
322@click.argument("files", nargs=-1, type=click.File())
323def merge(files: Iterable[io.TextIOBase]) -> None:
324 """Merge a set of Felis FILES.
326 This will expand out the felis FILES so that it is easy to
327 override values (using @Id), then normalize to a single
328 output.
329 """
330 graph = []
331 for file in files:
332 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
333 if "@graph" not in schema_obj:
334 schema_obj["@type"] = "felis:Schema"
335 schema_obj["@context"] = DEFAULT_CONTEXT
336 graph.extend(jsonld.flatten(schema_obj))
337 updated_map: MutableMapping[str, Any] = {}
338 for item in graph:
339 _id = item["@id"]
340 item_to_update = updated_map.get(_id, item)
341 if item_to_update and item_to_update != item:
342 logger.debug(f"Overwriting {_id}")
343 item_to_update.update(item)
344 updated_map[_id] = item_to_update
345 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
346 normalized = _normalize(merged, embed="@always")
347 _dump(normalized)
350@cli.command("validate")
351@click.option(
352 "-s",
353 "--schema-name",
354 help="Schema name for validation",
355 type=click.Choice(["RSP", "default"]),
356 default="default",
357)
358@click.option(
359 "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
360)
361@click.option(
362 "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
363)
364@click.argument("files", nargs=-1, type=click.File())
365def validate(
366 schema_name: str,
367 require_description: bool,
368 check_redundant_datatypes: bool,
369 files: Iterable[io.TextIOBase],
370) -> None:
371 """Validate one or more felis YAML files."""
372 schema_class = get_schema(schema_name)
373 if schema_name != "default":
374 logger.info(f"Using schema '{schema_class.__name__}'")
376 rc = 0
377 for file in files:
378 file_name = getattr(file, "name", None)
379 logger.info(f"Validating {file_name}")
380 try:
381 data = yaml.load(file, Loader=yaml.SafeLoader)
382 schema_class.model_validate(
383 data,
384 context={
385 "check_redundant_datatypes": check_redundant_datatypes,
386 "require_description": require_description,
387 },
388 )
389 except ValidationError as e:
390 logger.error(e)
391 rc = 1
392 if rc:
393 raise click.exceptions.Exit(rc)
396@cli.command("dump-json")
397@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
398@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
399@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
400@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
401@click.argument("file", type=click.File())
402def dump_json(
403 file: io.TextIOBase,
404 expanded: bool = False,
405 compacted: bool = False,
406 framed: bool = False,
407 graph: bool = False,
408) -> None:
409 """Dump JSON representation using various JSON-LD options."""
410 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
411 schema_obj["@type"] = "felis:Schema"
412 # Force Context and Schema Type
413 schema_obj["@context"] = DEFAULT_CONTEXT
415 if expanded:
416 schema_obj = jsonld.expand(schema_obj)
417 if framed:
418 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
419 if compacted:
420 options = {}
421 if graph:
422 options["graph"] = True
423 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
424 json.dump(schema_obj, sys.stdout, indent=4)
427def _dump(obj: Mapping[str, Any]) -> None:
428 class OrderedDumper(yaml.Dumper):
429 pass
431 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
432 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
434 OrderedDumper.add_representer(dict, _dict_representer)
435 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
438def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]:
439 framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed))
440 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
441 graph = compacted["@graph"]
442 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
443 compacted["@graph"] = graph if len(graph) > 1 else graph[0]
444 return compacted
447if __name__ == "__main__": 447 ↛ 448line 447 didn't jump to line 448, because the condition on line 447 was never true
448 cli()