Coverage for python/felis/cli.py: 30%
202 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-14 01:56 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-14 01:56 -0800
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import io
23import json
24import logging
25import sys
26from collections.abc import Iterable, Mapping, MutableMapping
27from typing import Any
29import click
30import yaml
31from pyld import jsonld
32from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
33from sqlalchemy.engine.mock import MockConnection
35from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
36from .check import CheckingVisitor
37from .sql import SQLVisitor
38from .tap import Tap11Base, TapLoadingVisitor, init_tables
39from .utils import ReorderingVisitor
41logger = logging.getLogger("felis")
44@click.group()
45@click.version_option(__version__)
46def cli() -> None:
47 """Felis Command Line Tools"""
48 logging.basicConfig(level=logging.INFO)
51@cli.command("create-all")
52@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
53@click.option("--schema-name", help="Alternate Schema Name for Felis File")
54@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
55@click.argument("file", type=click.File())
56def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
57 """Create schema objects from the Felis FILE."""
59 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
60 visitor = SQLVisitor(schema_name=schema_name)
61 schema = visitor.visit_schema(schema_obj)
63 metadata = schema.metadata
65 engine: Engine | MockConnection
66 if not dry_run:
67 engine = create_engine(engine_url)
68 else:
69 _insert_dump = InsertDump()
70 engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump)
71 _insert_dump.dialect = engine.dialect
72 metadata.create_all(engine)
75@cli.command("init-tap")
76@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
77@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
78@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
79@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
80@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
81@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
82@click.argument("engine-url")
83def init_tap(
84 engine_url: str,
85 tap_schema_name: str,
86 tap_schemas_table: str,
87 tap_tables_table: str,
88 tap_columns_table: str,
89 tap_keys_table: str,
90 tap_key_columns_table: str,
91) -> None:
92 """Initialize TAP 1.1 TAP_SCHEMA objects.
93 Please verify the schema/catalog you are executing this in in your
94 engine URL."""
95 engine = create_engine(engine_url, echo=True)
96 init_tables(
97 tap_schema_name,
98 tap_schemas_table,
99 tap_tables_table,
100 tap_columns_table,
101 tap_keys_table,
102 tap_key_columns_table,
103 )
104 Tap11Base.metadata.create_all(engine)
107@cli.command("load-tap")
108@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
109@click.option("--schema-name", help="Alternate Schema Name for Felis file")
110@click.option("--catalog-name", help="Catalog Name for Schema")
111@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
112@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
113@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
114@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
115@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
116@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
117@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
118@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
119@click.argument("file", type=click.File())
120def load_tap(
121 engine_url: str,
122 schema_name: str,
123 catalog_name: str,
124 dry_run: bool,
125 tap_schema_name: str,
126 tap_tables_postfix: str,
127 tap_schemas_table: str,
128 tap_tables_table: str,
129 tap_columns_table: str,
130 tap_keys_table: str,
131 tap_key_columns_table: str,
132 file: io.TextIOBase,
133) -> None:
134 """Load TAP metadata from a Felis FILE.
135 This command loads the associated TAP metadata from a Felis FILE
136 to the TAP_SCHEMA tables."""
137 top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
138 schema_obj: dict
139 if isinstance(top_level_object, dict):
140 schema_obj = top_level_object
141 if "@graph" not in schema_obj:
142 schema_obj["@type"] = "felis:Schema"
143 schema_obj["@context"] = DEFAULT_CONTEXT
144 elif isinstance(top_level_object, list):
145 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
146 else:
147 logger.error("Schema object not of recognizable type")
148 sys.exit(1)
150 normalized = _normalize(schema_obj)
151 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
152 logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
153 sys.exit(1)
155 # Force normalized["@graph"] to a list, which is what happens when there's
156 # multiple schemas
157 if isinstance(normalized["@graph"], dict):
158 normalized["@graph"] = [normalized["@graph"]]
160 if not dry_run:
161 engine = create_engine(engine_url)
162 else:
163 _insert_dump = InsertDump()
164 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump, paramstyle="pyformat")
165 # After the engine is created, update the executor with the dialect
166 _insert_dump.dialect = engine.dialect
167 tap_tables = init_tables(
168 tap_schema_name,
169 tap_tables_postfix,
170 tap_schemas_table,
171 tap_tables_table,
172 tap_columns_table,
173 tap_keys_table,
174 tap_key_columns_table,
175 )
177 if engine_url == "sqlite://" and not dry_run:
178 # In Memory SQLite - Mostly used to test
179 Tap11Base.metadata.create_all(engine)
181 for schema in normalized["@graph"]:
182 tap_visitor = TapLoadingVisitor(
183 engine, catalog_name=catalog_name, schema_name=schema_name, mock=dry_run, tap_tables=tap_tables
184 )
185 tap_visitor.visit_schema(schema)
188@cli.command("modify-tap")
189@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index")
190@click.argument("files", nargs=-1, type=click.File())
191def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
192 """Modify TAP information in Felis schema FILES.
193 This command has some utilities to aid in rewriting felis FILES
194 in specific ways. It will write out a merged version of these files.
195 """
196 count = 0
197 graph = []
198 for file in files:
199 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
200 if "@graph" not in schema_obj:
201 schema_obj["@type"] = "felis:Schema"
202 schema_obj["@context"] = DEFAULT_CONTEXT
203 schema_index = schema_obj.get("tap:schema_index")
204 if not schema_index or (schema_index and schema_index > start_schema_at):
205 schema_index = start_schema_at + count
206 count += 1
207 schema_obj["tap:schema_index"] = schema_index
208 graph.extend(jsonld.flatten(schema_obj))
209 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
210 normalized = _normalize(merged)
211 _dump(normalized)
214@cli.command("basic-check")
215@click.argument("file", type=click.File())
216def basic_check(file: io.TextIOBase) -> None:
217 """Perform a basic check on a felis FILE.
218 This performs a very check to ensure required fields are
219 populated and basic semantics are okay. It does not ensure semantics
220 are valid for other commands like create-all or load-tap.
221 """
222 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
223 schema_obj["@type"] = "felis:Schema"
224 # Force Context and Schema Type
225 schema_obj["@context"] = DEFAULT_CONTEXT
226 check_visitor = CheckingVisitor()
227 check_visitor.visit_schema(schema_obj)
230@cli.command("normalize")
231@click.argument("file", type=click.File())
232def normalize(file: io.TextIOBase) -> None:
233 """Normalize a Felis FILE.
234 Takes a felis schema FILE, expands it (resolving the full URLs),
235 then compacts it, and finally produces output in the canonical
236 format.
238 (This is most useful in some debugging scenarios)
240 See Also:
241 https://json-ld.org/spec/latest/json-ld/#expanded-document-form
242 https://json-ld.org/spec/latest/json-ld/#compacted-document-form
243 """
244 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
245 schema_obj["@type"] = "felis:Schema"
246 # Force Context and Schema Type
247 schema_obj["@context"] = DEFAULT_CONTEXT
248 expanded = jsonld.expand(schema_obj)
249 normalized = _normalize(expanded)
250 _dump(normalized)
253@cli.command("merge")
254@click.argument("files", nargs=-1, type=click.File())
255def merge(files: Iterable[io.TextIOBase]) -> None:
256 """Merge a set of Felis FILES.
258 This will expand out the felis FILES so that it is easy to
259 override values (using @Id), then normalize to a single
260 output.
261 """
262 graph = []
263 for file in files:
264 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
265 if "@graph" not in schema_obj:
266 schema_obj["@type"] = "felis:Schema"
267 schema_obj["@context"] = DEFAULT_CONTEXT
268 graph.extend(jsonld.flatten(schema_obj))
269 updated_map: MutableMapping[str, Any] = {}
270 for item in graph:
271 _id = item["@id"]
272 item_to_update = updated_map.get(_id, item)
273 if item_to_update and item_to_update != item:
274 logger.debug(f"Overwriting {_id}")
275 item_to_update.update(item)
276 updated_map[_id] = item_to_update
277 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
278 normalized = _normalize(merged)
279 _dump(normalized)
282@cli.command("dump-json")
283@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
284@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
285@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
286@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
287@click.argument("file", type=click.File())
288def dump_json(
289 file: io.TextIOBase,
290 expanded: bool = False,
291 compacted: bool = False,
292 framed: bool = False,
293 graph: bool = False,
294) -> None:
295 """Dump JSON representation using various JSON-LD options."""
296 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
297 schema_obj["@type"] = "felis:Schema"
298 # Force Context and Schema Type
299 schema_obj["@context"] = DEFAULT_CONTEXT
301 if expanded:
302 schema_obj = jsonld.expand(schema_obj)
303 if framed:
304 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
305 if compacted:
306 options = {}
307 if graph:
308 options["graph"] = True
309 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
310 json.dump(schema_obj, sys.stdout, indent=4)
313def _dump(obj: Mapping[str, Any]) -> None:
314 class OrderedDumper(yaml.Dumper):
315 pass
317 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
318 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
320 OrderedDumper.add_representer(dict, _dict_representer)
321 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
324def _normalize(schema_obj: Mapping[str, Any]) -> MutableMapping[str, Any]:
325 framed = jsonld.frame(schema_obj, DEFAULT_FRAME)
326 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
327 graph = compacted["@graph"]
328 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
329 compacted["@graph"] = graph if len(graph) > 1 else graph[0]
330 return compacted
333class InsertDump:
334 """An Insert Dumper for SQL statements"""
336 dialect: Any = None
338 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
339 compiled = sql.compile(dialect=self.dialect)
340 sql_str = str(compiled) + ";"
341 params_list = [compiled.params]
342 for params in params_list:
343 if not params:
344 print(sql_str)
345 continue
346 new_params = {}
347 for key, value in params.items():
348 if isinstance(value, str):
349 new_params[key] = f"'{value}'"
350 elif value is None:
351 new_params[key] = "null"
352 else:
353 new_params[key] = value
355 print(sql_str % new_params)
358if __name__ == "__main__": 358 ↛ 359line 358 didn't jump to line 359, because the condition on line 358 was never true
359 cli()