Coverage for python/felis/cli.py: 29%
201 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-22 01:55 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-22 01:55 -0700
1# This file is part of felis.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import io
23import json
24import logging
25import sys
26from collections.abc import Iterable, Mapping, MutableMapping
27from typing import Any
29import click
30import yaml
31from pyld import jsonld
32from sqlalchemy import create_engine
34from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
35from .check import CheckingVisitor
36from .sql import SQLVisitor
37from .tap import Tap11Base, TapLoadingVisitor, init_tables
38from .utils import ReorderingVisitor
40logger = logging.getLogger("felis")
43@click.group()
44@click.version_option(__version__)
45def cli() -> None:
46 """Felis Command Line Tools"""
47 logging.basicConfig(level=logging.INFO)
50@cli.command("create-all")
51@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
52@click.option("--schema-name", help="Alternate Schema Name for Felis File")
53@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
54@click.argument("file", type=click.File())
55def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
56 """Create schema objects from the Felis FILE."""
58 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
59 visitor = SQLVisitor(schema_name=schema_name)
60 schema = visitor.visit_schema(schema_obj)
62 metadata = schema.metadata
64 if not dry_run:
65 engine = create_engine(engine_url)
66 else:
67 _insert_dump = InsertDump()
68 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump)
69 _insert_dump.dialect = engine.dialect
70 metadata.create_all(engine)
73@cli.command("init-tap")
74@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
75@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
76@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
77@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
78@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
79@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
80@click.argument("engine-url")
81def init_tap(
82 engine_url: str,
83 tap_schema_name: str,
84 tap_schemas_table: str,
85 tap_tables_table: str,
86 tap_columns_table: str,
87 tap_keys_table: str,
88 tap_key_columns_table: str,
89) -> None:
90 """Initialize TAP 1.1 TAP_SCHEMA objects.
91 Please verify the schema/catalog you are executing this in in your
92 engine URL."""
93 engine = create_engine(engine_url, echo=True)
94 init_tables(
95 tap_schema_name,
96 tap_schemas_table,
97 tap_tables_table,
98 tap_columns_table,
99 tap_keys_table,
100 tap_key_columns_table,
101 )
102 Tap11Base.metadata.create_all(engine)
105@cli.command("load-tap")
106@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
107@click.option("--schema-name", help="Alternate Schema Name for Felis file")
108@click.option("--catalog-name", help="Catalog Name for Schema")
109@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
110@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
111@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
112@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
113@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
114@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
115@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
116@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
117@click.argument("file", type=click.File())
118def load_tap(
119 engine_url: str,
120 schema_name: str,
121 catalog_name: str,
122 dry_run: bool,
123 tap_schema_name: str,
124 tap_tables_postfix: str,
125 tap_schemas_table: str,
126 tap_tables_table: str,
127 tap_columns_table: str,
128 tap_keys_table: str,
129 tap_key_columns_table: str,
130 file: io.TextIOBase,
131) -> None:
132 """Load TAP metadata from a Felis FILE.
133 This command loads the associated TAP metadata from a Felis FILE
134 to the TAP_SCHEMA tables."""
135 top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
136 schema_obj: dict
137 if isinstance(top_level_object, dict):
138 schema_obj = top_level_object
139 if "@graph" not in schema_obj:
140 schema_obj["@type"] = "felis:Schema"
141 schema_obj["@context"] = DEFAULT_CONTEXT
142 elif isinstance(top_level_object, list):
143 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
144 else:
145 logger.error("Schema object not of recognizable type")
146 sys.exit(1)
148 normalized = _normalize(schema_obj)
149 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
150 logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
151 sys.exit(1)
153 # Force normalized["@graph"] to a list, which is what happens when there's
154 # multiple schemas
155 if isinstance(normalized["@graph"], dict):
156 normalized["@graph"] = [normalized["@graph"]]
158 if not dry_run:
159 engine = create_engine(engine_url)
160 else:
161 _insert_dump = InsertDump()
162 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump, paramstyle="pyformat")
163 # After the engine is created, update the executor with the dialect
164 _insert_dump.dialect = engine.dialect
165 tap_tables = init_tables(
166 tap_schema_name,
167 tap_tables_postfix,
168 tap_schemas_table,
169 tap_tables_table,
170 tap_columns_table,
171 tap_keys_table,
172 tap_key_columns_table,
173 )
175 if engine_url == "sqlite://" and not dry_run:
176 # In Memory SQLite - Mostly used to test
177 Tap11Base.metadata.create_all(engine)
179 for schema in normalized["@graph"]:
180 tap_visitor = TapLoadingVisitor(
181 engine, catalog_name=catalog_name, schema_name=schema_name, mock=dry_run, tap_tables=tap_tables
182 )
183 tap_visitor.visit_schema(schema)
186@cli.command("modify-tap")
187@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index")
188@click.argument("files", nargs=-1, type=click.File())
189def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
190 """Modify TAP information in Felis schema FILES.
191 This command has some utilities to aid in rewriting felis FILES
192 in specific ways. It will write out a merged version of these files.
193 """
194 count = 0
195 graph = []
196 for file in files:
197 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
198 if "@graph" not in schema_obj:
199 schema_obj["@type"] = "felis:Schema"
200 schema_obj["@context"] = DEFAULT_CONTEXT
201 schema_index = schema_obj.get("tap:schema_index")
202 if not schema_index or (schema_index and schema_index > start_schema_at):
203 schema_index = start_schema_at + count
204 count += 1
205 schema_obj["tap:schema_index"] = schema_index
206 graph.extend(jsonld.flatten(schema_obj))
207 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
208 normalized = _normalize(merged)
209 _dump(normalized)
212@cli.command("basic-check")
213@click.argument("file", type=click.File())
214def basic_check(file: io.TextIOBase) -> None:
215 """Perform a basic check on a felis FILE.
216 This performs a very check to ensure required fields are
217 populated and basic semantics are okay. It does not ensure semantics
218 are valid for other commands like create-all or load-tap.
219 """
220 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
221 schema_obj["@type"] = "felis:Schema"
222 # Force Context and Schema Type
223 schema_obj["@context"] = DEFAULT_CONTEXT
224 check_visitor = CheckingVisitor()
225 check_visitor.visit_schema(schema_obj)
228@cli.command("normalize")
229@click.argument("file", type=click.File())
230def normalize(file: io.TextIOBase) -> None:
231 """Normalize a Felis FILE.
232 Takes a felis schema FILE, expands it (resolving the full URLs),
233 then compacts it, and finally produces output in the canonical
234 format.
236 (This is most useful in some debugging scenarios)
238 See Also:
239 https://json-ld.org/spec/latest/json-ld/#expanded-document-form
240 https://json-ld.org/spec/latest/json-ld/#compacted-document-form
241 """
242 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
243 schema_obj["@type"] = "felis:Schema"
244 # Force Context and Schema Type
245 schema_obj["@context"] = DEFAULT_CONTEXT
246 expanded = jsonld.expand(schema_obj)
247 normalized = _normalize(expanded)
248 _dump(normalized)
251@cli.command("merge")
252@click.argument("files", nargs=-1, type=click.File())
253def merge(files: Iterable[io.TextIOBase]) -> None:
254 """Merge a set of Felis FILES.
256 This will expand out the felis FILES so that it is easy to
257 override values (using @Id), then normalize to a single
258 output.
259 """
260 graph = []
261 for file in files:
262 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
263 if "@graph" not in schema_obj:
264 schema_obj["@type"] = "felis:Schema"
265 schema_obj["@context"] = DEFAULT_CONTEXT
266 graph.extend(jsonld.flatten(schema_obj))
267 updated_map: MutableMapping[str, Any] = {}
268 for item in graph:
269 _id = item["@id"]
270 item_to_update = updated_map.get(_id, item)
271 if item_to_update and item_to_update != item:
272 logger.debug(f"Overwriting {_id}")
273 item_to_update.update(item)
274 updated_map[_id] = item_to_update
275 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
276 normalized = _normalize(merged)
277 _dump(normalized)
280@cli.command("dump-json")
281@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
282@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
283@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
284@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
285@click.argument("file", type=click.File())
286def dump_json(
287 file: io.TextIOBase,
288 expanded: bool = False,
289 compacted: bool = False,
290 framed: bool = False,
291 graph: bool = False,
292) -> None:
293 """Dump JSON representation using various JSON-LD options."""
294 schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
295 schema_obj["@type"] = "felis:Schema"
296 # Force Context and Schema Type
297 schema_obj["@context"] = DEFAULT_CONTEXT
299 if expanded:
300 schema_obj = jsonld.expand(schema_obj)
301 if framed:
302 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
303 if compacted:
304 options = {}
305 if graph:
306 options["graph"] = True
307 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
308 json.dump(schema_obj, sys.stdout, indent=4)
311def _dump(obj: Mapping[str, Any]) -> None:
312 class OrderedDumper(yaml.Dumper):
313 pass
315 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
316 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
318 OrderedDumper.add_representer(dict, _dict_representer)
319 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
322def _normalize(schema_obj: Mapping[str, Any]) -> MutableMapping[str, Any]:
323 framed = jsonld.frame(schema_obj, DEFAULT_FRAME)
324 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
325 graph = compacted["@graph"]
326 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
327 compacted["@graph"] = graph if len(graph) > 1 else graph[0]
328 return compacted
331class InsertDump:
332 """An Insert Dumper for SQL statements"""
334 dialect: Any = None
336 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
337 compiled = sql.compile(dialect=self.dialect)
338 sql_str = str(compiled) + ";"
339 params_list = [compiled.params]
340 for params in params_list:
341 if not params:
342 print(sql_str)
343 continue
344 new_params = {}
345 for key, value in params.items():
346 if isinstance(value, str):
347 new_params[key] = f"'{value}'"
348 elif value is None:
349 new_params[key] = "null"
350 else:
351 new_params[key] = value
353 print(sql_str % new_params)
356if __name__ == "__main__": 356 ↛ 357line 356 didn't jump to line 357, because the condition on line 356 was never true
357 cli()