Coverage for python/felis/cli.py: 29%
205 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 02:20 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 02:20 -0700
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import io
23import json
24import logging
25import sys
26from collections.abc import Iterable, Mapping, MutableMapping
27from typing import Any
29import click
30import yaml
31from pyld import jsonld
32from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
33from sqlalchemy.engine.mock import MockConnection
35from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
36from .check import CheckingVisitor
37from .sql import SQLVisitor
38from .tap import Tap11Base, TapLoadingVisitor, init_tables
39from .utils import ReorderingVisitor
41logger = logging.getLogger("felis")
44@click.group()
45@click.version_option(__version__)
46def cli() -> None:
47 """Felis Command Line Tools"""
48 logging.basicConfig(level=logging.INFO)
51@cli.command("create-all")
52@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
53@click.option("--schema-name", help="Alternate Schema Name for Felis File")
54@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
55@click.argument("file", type=click.File())
56def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
57 """Create schema objects from the Felis FILE."""
59 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
60 visitor = SQLVisitor(schema_name=schema_name)
61 schema = visitor.visit_schema(schema_obj)
63 metadata = schema.metadata
65 engine: Engine | MockConnection
66 if not dry_run:
67 engine = create_engine(engine_url)
68 else:
69 _insert_dump = InsertDump()
70 engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump)
71 _insert_dump.dialect = engine.dialect
72 metadata.create_all(engine)
75@cli.command("init-tap")
76@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
77@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
78@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
79@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
80@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
81@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
82@click.argument("engine-url")
83def init_tap(
84 engine_url: str,
85 tap_schema_name: str,
86 tap_schemas_table: str,
87 tap_tables_table: str,
88 tap_columns_table: str,
89 tap_keys_table: str,
90 tap_key_columns_table: str,
91) -> None:
92 """Initialize TAP 1.1 TAP_SCHEMA objects.
93 Please verify the schema/catalog you are executing this in in your
94 engine URL."""
95 engine = create_engine(engine_url, echo=True)
96 init_tables(
97 tap_schema_name,
98 tap_schemas_table,
99 tap_tables_table,
100 tap_columns_table,
101 tap_keys_table,
102 tap_key_columns_table,
103 )
104 Tap11Base.metadata.create_all(engine)
107@cli.command("load-tap")
108@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
109@click.option("--schema-name", help="Alternate Schema Name for Felis file")
110@click.option("--catalog-name", help="Catalog Name for Schema")
111@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
112@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
113@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
114@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
115@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
116@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
117@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
118@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
119@click.argument("file", type=click.File())
120def load_tap(
121 engine_url: str,
122 schema_name: str,
123 catalog_name: str,
124 dry_run: bool,
125 tap_schema_name: str,
126 tap_tables_postfix: str,
127 tap_schemas_table: str,
128 tap_tables_table: str,
129 tap_columns_table: str,
130 tap_keys_table: str,
131 tap_key_columns_table: str,
132 file: io.TextIOBase,
133) -> None:
134 """Load TAP metadata from a Felis FILE.
135 This command loads the associated TAP metadata from a Felis FILE
136 to the TAP_SCHEMA tables."""
137 top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
138 schema_obj: dict
139 if isinstance(top_level_object, dict):
140 schema_obj = top_level_object
141 if "@graph" not in schema_obj:
142 schema_obj["@type"] = "felis:Schema"
143 schema_obj["@context"] = DEFAULT_CONTEXT
144 elif isinstance(top_level_object, list):
145 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
146 else:
147 logger.error("Schema object not of recognizable type")
148 sys.exit(1)
150 normalized = _normalize(schema_obj)
151 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
152 logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
153 sys.exit(1)
155 # Force normalized["@graph"] to a list, which is what happens when there's
156 # multiple schemas
157 if isinstance(normalized["@graph"], dict):
158 normalized["@graph"] = [normalized["@graph"]]
160 tap_tables = init_tables(
161 tap_schema_name,
162 tap_tables_postfix,
163 tap_schemas_table,
164 tap_tables_table,
165 tap_columns_table,
166 tap_keys_table,
167 tap_key_columns_table,
168 )
170 if not dry_run:
171 engine = create_engine(engine_url)
173 if engine_url == "sqlite://" and not dry_run:
174 # In Memory SQLite - Mostly used to test
175 Tap11Base.metadata.create_all(engine)
177 for schema in normalized["@graph"]:
178 tap_visitor = TapLoadingVisitor(
179 engine,
180 catalog_name=catalog_name,
181 schema_name=schema_name,
182 tap_tables=tap_tables,
183 )
184 tap_visitor.visit_schema(schema)
185 else:
186 _insert_dump = InsertDump()
187 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
188 # After the engine is created, update the executor with the dialect
189 _insert_dump.dialect = conn.dialect
191 for schema in normalized["@graph"]:
192 tap_visitor = TapLoadingVisitor.from_mock_connection(
193 conn,
194 catalog_name=catalog_name,
195 schema_name=schema_name,
196 tap_tables=tap_tables,
197 )
198 tap_visitor.visit_schema(schema)
201@cli.command("modify-tap")
202@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index")
203@click.argument("files", nargs=-1, type=click.File())
204def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
205 """Modify TAP information in Felis schema FILES.
206 This command has some utilities to aid in rewriting felis FILES
207 in specific ways. It will write out a merged version of these files.
208 """
209 count = 0
210 graph = []
211 for file in files:
212 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
213 if "@graph" not in schema_obj:
214 schema_obj["@type"] = "felis:Schema"
215 schema_obj["@context"] = DEFAULT_CONTEXT
216 schema_index = schema_obj.get("tap:schema_index")
217 if not schema_index or (schema_index and schema_index > start_schema_at):
218 schema_index = start_schema_at + count
219 count += 1
220 schema_obj["tap:schema_index"] = schema_index
221 graph.extend(jsonld.flatten(schema_obj))
222 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
223 normalized = _normalize(merged)
224 _dump(normalized)
227@cli.command("basic-check")
228@click.argument("file", type=click.File())
229def basic_check(file: io.TextIOBase) -> None:
230 """Perform a basic check on a felis FILE.
231 This performs a very check to ensure required fields are
232 populated and basic semantics are okay. It does not ensure semantics
233 are valid for other commands like create-all or load-tap.
234 """
235 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
236 schema_obj["@type"] = "felis:Schema"
237 # Force Context and Schema Type
238 schema_obj["@context"] = DEFAULT_CONTEXT
239 check_visitor = CheckingVisitor()
240 check_visitor.visit_schema(schema_obj)
243@cli.command("normalize")
244@click.argument("file", type=click.File())
245def normalize(file: io.TextIOBase) -> None:
246 """Normalize a Felis FILE.
247 Takes a felis schema FILE, expands it (resolving the full URLs),
248 then compacts it, and finally produces output in the canonical
249 format.
251 (This is most useful in some debugging scenarios)
253 See Also:
254 https://json-ld.org/spec/latest/json-ld/#expanded-document-form
255 https://json-ld.org/spec/latest/json-ld/#compacted-document-form
256 """
257 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
258 schema_obj["@type"] = "felis:Schema"
259 # Force Context and Schema Type
260 schema_obj["@context"] = DEFAULT_CONTEXT
261 expanded = jsonld.expand(schema_obj)
262 normalized = _normalize(expanded)
263 _dump(normalized)
266@cli.command("merge")
267@click.argument("files", nargs=-1, type=click.File())
268def merge(files: Iterable[io.TextIOBase]) -> None:
269 """Merge a set of Felis FILES.
271 This will expand out the felis FILES so that it is easy to
272 override values (using @Id), then normalize to a single
273 output.
274 """
275 graph = []
276 for file in files:
277 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
278 if "@graph" not in schema_obj:
279 schema_obj["@type"] = "felis:Schema"
280 schema_obj["@context"] = DEFAULT_CONTEXT
281 graph.extend(jsonld.flatten(schema_obj))
282 updated_map: MutableMapping[str, Any] = {}
283 for item in graph:
284 _id = item["@id"]
285 item_to_update = updated_map.get(_id, item)
286 if item_to_update and item_to_update != item:
287 logger.debug(f"Overwriting {_id}")
288 item_to_update.update(item)
289 updated_map[_id] = item_to_update
290 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
291 normalized = _normalize(merged)
292 _dump(normalized)
295@cli.command("dump-json")
296@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
297@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
298@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
299@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
300@click.argument("file", type=click.File())
301def dump_json(
302 file: io.TextIOBase,
303 expanded: bool = False,
304 compacted: bool = False,
305 framed: bool = False,
306 graph: bool = False,
307) -> None:
308 """Dump JSON representation using various JSON-LD options."""
309 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
310 schema_obj["@type"] = "felis:Schema"
311 # Force Context and Schema Type
312 schema_obj["@context"] = DEFAULT_CONTEXT
314 if expanded:
315 schema_obj = jsonld.expand(schema_obj)
316 if framed:
317 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
318 if compacted:
319 options = {}
320 if graph:
321 options["graph"] = True
322 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
323 json.dump(schema_obj, sys.stdout, indent=4)
326def _dump(obj: Mapping[str, Any]) -> None:
327 class OrderedDumper(yaml.Dumper):
328 pass
330 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
331 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
333 OrderedDumper.add_representer(dict, _dict_representer)
334 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
337def _normalize(schema_obj: Mapping[str, Any]) -> MutableMapping[str, Any]:
338 framed = jsonld.frame(schema_obj, DEFAULT_FRAME)
339 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
340 graph = compacted["@graph"]
341 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
342 compacted["@graph"] = graph if len(graph) > 1 else graph[0]
343 return compacted
346class InsertDump:
347 """An Insert Dumper for SQL statements"""
349 dialect: Any = None
351 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
352 compiled = sql.compile(dialect=self.dialect)
353 sql_str = str(compiled) + ";"
354 params_list = [compiled.params]
355 for params in params_list:
356 if not params:
357 print(sql_str)
358 continue
359 new_params = {}
360 for key, value in params.items():
361 if isinstance(value, str):
362 new_params[key] = f"'{value}'"
363 elif value is None:
364 new_params[key] = "null"
365 else:
366 new_params[key] = value
368 print(sql_str % new_params)
371if __name__ == "__main__": 371 ↛ 372line 371 didn't jump to line 372, because the condition on line 371 was never true
372 cli()