Coverage for python/felis/cli.py: 50%
234 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 03:38 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 03:38 -0700
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from __future__ import annotations
24import io
25import json
26import logging
27import sys
28from collections.abc import Iterable, Mapping, MutableMapping
29from typing import Any
31import click
32import yaml
33from pydantic import ValidationError
34from pyld import jsonld
35from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
36from sqlalchemy.engine.mock import MockConnection
38from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
39from .check import CheckingVisitor
40from .datamodel import Schema
41from .sql import SQLVisitor
42from .tap import Tap11Base, TapLoadingVisitor, init_tables
43from .utils import ReorderingVisitor
44from .validation import get_schema
46logger = logging.getLogger("felis")
48loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
51@click.group()
52@click.version_option(__version__)
53@click.option(
54 "--log-level",
55 type=click.Choice(loglevel_choices),
56 envvar="FELIS_LOGLEVEL",
57 help="Felis log level",
58 default=logging.getLevelName(logging.INFO),
59)
60@click.option(
61 "--log-file",
62 type=click.Path(),
63 envvar="FELIS_LOGFILE",
64 help="Felis log file path",
65)
66def cli(log_level: str, log_file: str | None) -> None:
67 """Felis Command Line Tools."""
68 if log_file:
69 logging.basicConfig(filename=log_file, level=log_level)
70 else:
71 logging.basicConfig(level=log_level)
74@cli.command("create-all")
75@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
76@click.option("--schema-name", help="Alternate Schema Name for Felis File")
77@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
78@click.argument("file", type=click.File())
79def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
80 """Create schema objects from the Felis FILE."""
81 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
82 visitor = SQLVisitor(schema_name=schema_name)
83 schema = visitor.visit_schema(schema_obj)
85 metadata = schema.metadata
87 engine: Engine | MockConnection
88 if not dry_run:
89 engine = create_engine(engine_url)
90 else:
91 _insert_dump = InsertDump()
92 engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump)
93 _insert_dump.dialect = engine.dialect
94 metadata.create_all(engine)
97@cli.command("init-tap")
98@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
99@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
100@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
101@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
102@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
103@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
104@click.argument("engine-url")
105def init_tap(
106 engine_url: str,
107 tap_schema_name: str,
108 tap_schemas_table: str,
109 tap_tables_table: str,
110 tap_columns_table: str,
111 tap_keys_table: str,
112 tap_key_columns_table: str,
113) -> None:
114 """Initialize TAP 1.1 TAP_SCHEMA objects.
116 Please verify the schema/catalog you are executing this in in your
117 engine URL.
118 """
119 engine = create_engine(engine_url, echo=True)
120 init_tables(
121 tap_schema_name,
122 tap_schemas_table,
123 tap_tables_table,
124 tap_columns_table,
125 tap_keys_table,
126 tap_key_columns_table,
127 )
128 Tap11Base.metadata.create_all(engine)
131@cli.command("load-tap")
132@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
133@click.option("--schema-name", help="Alternate Schema Name for Felis file")
134@click.option("--catalog-name", help="Catalog Name for Schema")
135@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
136@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
137@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
138@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
139@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
140@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
141@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
142@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
143@click.argument("file", type=click.File())
144def load_tap(
145 engine_url: str,
146 schema_name: str,
147 catalog_name: str,
148 dry_run: bool,
149 tap_schema_name: str,
150 tap_tables_postfix: str,
151 tap_schemas_table: str,
152 tap_tables_table: str,
153 tap_columns_table: str,
154 tap_keys_table: str,
155 tap_key_columns_table: str,
156 file: io.TextIOBase,
157) -> None:
158 """Load TAP metadata from a Felis FILE.
160 This command loads the associated TAP metadata from a Felis FILE
161 to the TAP_SCHEMA tables.
162 """
163 top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
164 schema_obj: dict
165 if isinstance(top_level_object, dict):
166 schema_obj = top_level_object
167 if "@graph" not in schema_obj:
168 schema_obj["@type"] = "felis:Schema"
169 schema_obj["@context"] = DEFAULT_CONTEXT
170 elif isinstance(top_level_object, list):
171 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
172 else:
173 logger.error("Schema object not of recognizable type")
174 raise click.exceptions.Exit(1)
176 normalized = _normalize(schema_obj, embed="@always")
177 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
178 logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
179 raise click.exceptions.Exit(1)
181 # Force normalized["@graph"] to a list, which is what happens when there's
182 # multiple schemas
183 if isinstance(normalized["@graph"], dict):
184 normalized["@graph"] = [normalized["@graph"]]
186 tap_tables = init_tables(
187 tap_schema_name,
188 tap_tables_postfix,
189 tap_schemas_table,
190 tap_tables_table,
191 tap_columns_table,
192 tap_keys_table,
193 tap_key_columns_table,
194 )
196 if not dry_run:
197 engine = create_engine(engine_url)
199 if engine_url == "sqlite://" and not dry_run:
200 # In Memory SQLite - Mostly used to test
201 Tap11Base.metadata.create_all(engine)
203 for schema in normalized["@graph"]:
204 tap_visitor = TapLoadingVisitor(
205 engine,
206 catalog_name=catalog_name,
207 schema_name=schema_name,
208 tap_tables=tap_tables,
209 )
210 tap_visitor.visit_schema(schema)
211 else:
212 _insert_dump = InsertDump()
213 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
214 # After the engine is created, update the executor with the dialect
215 _insert_dump.dialect = conn.dialect
217 for schema in normalized["@graph"]:
218 tap_visitor = TapLoadingVisitor.from_mock_connection(
219 conn,
220 catalog_name=catalog_name,
221 schema_name=schema_name,
222 tap_tables=tap_tables,
223 )
224 tap_visitor.visit_schema(schema)
227@cli.command("modify-tap")
228@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0)
229@click.argument("files", nargs=-1, type=click.File())
230def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
231 """Modify TAP information in Felis schema FILES.
233 This command has some utilities to aid in rewriting felis FILES
234 in specific ways. It will write out a merged version of these files.
235 """
236 count = 0
237 graph = []
238 for file in files:
239 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
240 if "@graph" not in schema_obj:
241 schema_obj["@type"] = "felis:Schema"
242 schema_obj["@context"] = DEFAULT_CONTEXT
243 schema_index = schema_obj.get("tap:schema_index")
244 if not schema_index or (schema_index and schema_index > start_schema_at):
245 schema_index = start_schema_at + count
246 count += 1
247 schema_obj["tap:schema_index"] = schema_index
248 graph.extend(jsonld.flatten(schema_obj))
249 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
250 normalized = _normalize(merged, embed="@always")
251 _dump(normalized)
254@cli.command("basic-check")
255@click.argument("file", type=click.File())
256def basic_check(file: io.TextIOBase) -> None:
257 """Perform a basic check on a felis FILE.
259 This performs a very check to ensure required fields are
260 populated and basic semantics are okay. It does not ensure semantics
261 are valid for other commands like create-all or load-tap.
262 """
263 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
264 schema_obj["@type"] = "felis:Schema"
265 # Force Context and Schema Type
266 schema_obj["@context"] = DEFAULT_CONTEXT
267 check_visitor = CheckingVisitor()
268 check_visitor.visit_schema(schema_obj)
271@cli.command("normalize")
272@click.argument("file", type=click.File())
273def normalize(file: io.TextIOBase) -> None:
274 """Normalize a Felis FILE.
276 Takes a felis schema FILE, expands it (resolving the full URLs),
277 then compacts it, and finally produces output in the canonical
278 format.
280 (This is most useful in some debugging scenarios)
282 See Also :
284 https://json-ld.org/spec/latest/json-ld/#expanded-document-form
285 https://json-ld.org/spec/latest/json-ld/#compacted-document-form
286 """
287 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
288 schema_obj["@type"] = "felis:Schema"
289 # Force Context and Schema Type
290 schema_obj["@context"] = DEFAULT_CONTEXT
291 expanded = jsonld.expand(schema_obj)
292 normalized = _normalize(expanded, embed="@always")
293 _dump(normalized)
296@cli.command("merge")
297@click.argument("files", nargs=-1, type=click.File())
298def merge(files: Iterable[io.TextIOBase]) -> None:
299 """Merge a set of Felis FILES.
301 This will expand out the felis FILES so that it is easy to
302 override values (using @Id), then normalize to a single
303 output.
304 """
305 graph = []
306 for file in files:
307 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
308 if "@graph" not in schema_obj:
309 schema_obj["@type"] = "felis:Schema"
310 schema_obj["@context"] = DEFAULT_CONTEXT
311 graph.extend(jsonld.flatten(schema_obj))
312 updated_map: MutableMapping[str, Any] = {}
313 for item in graph:
314 _id = item["@id"]
315 item_to_update = updated_map.get(_id, item)
316 if item_to_update and item_to_update != item:
317 logger.debug(f"Overwriting {_id}")
318 item_to_update.update(item)
319 updated_map[_id] = item_to_update
320 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
321 normalized = _normalize(merged, embed="@always")
322 _dump(normalized)
325@cli.command("validate")
326@click.option(
327 "-s",
328 "--schema-name",
329 help="Schema name for validation",
330 type=click.Choice(["RSP", "default"]),
331 default="default",
332)
333@click.option("-d", "--require-description", is_flag=True, help="Require description for all objects")
334@click.argument("files", nargs=-1, type=click.File())
335def validate(schema_name: str, require_description: bool, files: Iterable[io.TextIOBase]) -> None:
336 """Validate one or more felis YAML files."""
337 schema_class = get_schema(schema_name)
338 logger.info(f"Using schema '{schema_class.__name__}'")
340 if require_description:
341 Schema.require_description(True)
343 rc = 0
344 for file in files:
345 file_name = getattr(file, "name", None)
346 logger.info(f"Validating {file_name}")
347 try:
348 schema_class.model_validate(yaml.load(file, Loader=yaml.SafeLoader))
349 except ValidationError as e:
350 logger.error(e)
351 rc = 1
352 if rc:
353 raise click.exceptions.Exit(rc)
356@cli.command("dump-json")
357@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
358@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
359@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
360@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
361@click.argument("file", type=click.File())
362def dump_json(
363 file: io.TextIOBase,
364 expanded: bool = False,
365 compacted: bool = False,
366 framed: bool = False,
367 graph: bool = False,
368) -> None:
369 """Dump JSON representation using various JSON-LD options."""
370 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
371 schema_obj["@type"] = "felis:Schema"
372 # Force Context and Schema Type
373 schema_obj["@context"] = DEFAULT_CONTEXT
375 if expanded:
376 schema_obj = jsonld.expand(schema_obj)
377 if framed:
378 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
379 if compacted:
380 options = {}
381 if graph:
382 options["graph"] = True
383 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
384 json.dump(schema_obj, sys.stdout, indent=4)
387def _dump(obj: Mapping[str, Any]) -> None:
388 class OrderedDumper(yaml.Dumper):
389 pass
391 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
392 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
394 OrderedDumper.add_representer(dict, _dict_representer)
395 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
398def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]:
399 framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed))
400 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
401 graph = compacted["@graph"]
402 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
403 compacted["@graph"] = graph if len(graph) > 1 else graph[0]
404 return compacted
407class InsertDump:
408 """An Insert Dumper for SQL statements."""
410 dialect: Any = None
412 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
413 compiled = sql.compile(dialect=self.dialect)
414 sql_str = str(compiled) + ";"
415 params_list = [compiled.params]
416 for params in params_list:
417 if not params:
418 print(sql_str)
419 continue
420 new_params = {}
421 for key, value in params.items():
422 if isinstance(value, str):
423 new_params[key] = f"'{value}'"
424 elif value is None:
425 new_params[key] = "null"
426 else:
427 new_params[key] = value
429 print(sql_str % new_params)
432if __name__ == "__main__": 432 ↛ 433line 432 didn't jump to line 433, because the condition on line 432 was never true
433 cli()