Coverage for python/felis/cli.py: 50%

234 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-14 10:16 -0700

1# This file is part of felis. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import io 

25import json 

26import logging 

27import sys 

28from collections.abc import Iterable, Mapping, MutableMapping 

29from typing import Any 

30 

31import click 

32import yaml 

33from pydantic import ValidationError 

34from pyld import jsonld 

35from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url 

36from sqlalchemy.engine.mock import MockConnection 

37 

38from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__ 

39from .check import CheckingVisitor 

40from .datamodel import Schema 

41from .sql import SQLVisitor 

42from .tap import Tap11Base, TapLoadingVisitor, init_tables 

43from .utils import ReorderingVisitor 

44from .validation import get_schema 

45 

46logger = logging.getLogger("felis") 

47 

48loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"] 

49 

50 

51@click.group() 

52@click.version_option(__version__) 

53@click.option( 

54 "--log-level", 

55 type=click.Choice(loglevel_choices), 

56 envvar="FELIS_LOGLEVEL", 

57 help="Felis log level", 

58 default=logging.getLevelName(logging.INFO), 

59) 

60@click.option( 

61 "--log-file", 

62 type=click.Path(), 

63 envvar="FELIS_LOGFILE", 

64 help="Felis log file path", 

65) 

66def cli(log_level: str, log_file: str | None) -> None: 

67 """Felis Command Line Tools.""" 

68 if log_file: 

69 logging.basicConfig(filename=log_file, level=log_level) 

70 else: 

71 logging.basicConfig(level=log_level) 

72 

73 

74@cli.command("create-all") 

75@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL") 

76@click.option("--schema-name", help="Alternate Schema Name for Felis File") 

77@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

78@click.argument("file", type=click.File()) 

79def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None: 

80 """Create schema objects from the Felis FILE.""" 

81 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

82 visitor = SQLVisitor(schema_name=schema_name) 

83 schema = visitor.visit_schema(schema_obj) 

84 

85 metadata = schema.metadata 

86 

87 engine: Engine | MockConnection 

88 if not dry_run: 

89 engine = create_engine(engine_url) 

90 else: 

91 _insert_dump = InsertDump() 

92 engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump) 

93 _insert_dump.dialect = engine.dialect 

94 metadata.create_all(engine) 

95 

96 

97@cli.command("init-tap") 

98@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

99@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

100@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

101@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

102@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

103@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

104@click.argument("engine-url") 

105def init_tap( 

106 engine_url: str, 

107 tap_schema_name: str, 

108 tap_schemas_table: str, 

109 tap_tables_table: str, 

110 tap_columns_table: str, 

111 tap_keys_table: str, 

112 tap_key_columns_table: str, 

113) -> None: 

114 """Initialize TAP 1.1 TAP_SCHEMA objects. 

115 

116 Please verify the schema/catalog you are executing this in in your 

117 engine URL. 

118 """ 

119 engine = create_engine(engine_url, echo=True) 

120 init_tables( 

121 tap_schema_name, 

122 tap_schemas_table, 

123 tap_tables_table, 

124 tap_columns_table, 

125 tap_keys_table, 

126 tap_key_columns_table, 

127 ) 

128 Tap11Base.metadata.create_all(engine) 

129 

130 

131@cli.command("load-tap") 

132@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog") 

133@click.option("--schema-name", help="Alternate Schema Name for Felis file") 

134@click.option("--catalog-name", help="Catalog Name for Schema") 

135@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

136@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

137@click.option("--tap-tables-postfix", help="Postfix for TAP table names") 

138@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

139@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

140@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

141@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

142@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

143@click.argument("file", type=click.File()) 

144def load_tap( 

145 engine_url: str, 

146 schema_name: str, 

147 catalog_name: str, 

148 dry_run: bool, 

149 tap_schema_name: str, 

150 tap_tables_postfix: str, 

151 tap_schemas_table: str, 

152 tap_tables_table: str, 

153 tap_columns_table: str, 

154 tap_keys_table: str, 

155 tap_key_columns_table: str, 

156 file: io.TextIOBase, 

157) -> None: 

158 """Load TAP metadata from a Felis FILE. 

159 

160 This command loads the associated TAP metadata from a Felis FILE 

161 to the TAP_SCHEMA tables. 

162 """ 

163 top_level_object = yaml.load(file, Loader=yaml.SafeLoader) 

164 schema_obj: dict 

165 if isinstance(top_level_object, dict): 

166 schema_obj = top_level_object 

167 if "@graph" not in schema_obj: 

168 schema_obj["@type"] = "felis:Schema" 

169 schema_obj["@context"] = DEFAULT_CONTEXT 

170 elif isinstance(top_level_object, list): 

171 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object} 

172 else: 

173 logger.error("Schema object not of recognizable type") 

174 raise click.exceptions.Exit(1) 

175 

176 normalized = _normalize(schema_obj, embed="@always") 

177 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name): 

178 logger.error("--schema-name and --catalog-name incompatible with multiple schemas") 

179 raise click.exceptions.Exit(1) 

180 

181 # Force normalized["@graph"] to a list, which is what happens when there's 

182 # multiple schemas 

183 if isinstance(normalized["@graph"], dict): 

184 normalized["@graph"] = [normalized["@graph"]] 

185 

186 tap_tables = init_tables( 

187 tap_schema_name, 

188 tap_tables_postfix, 

189 tap_schemas_table, 

190 tap_tables_table, 

191 tap_columns_table, 

192 tap_keys_table, 

193 tap_key_columns_table, 

194 ) 

195 

196 if not dry_run: 

197 engine = create_engine(engine_url) 

198 

199 if engine_url == "sqlite://" and not dry_run: 

200 # In Memory SQLite - Mostly used to test 

201 Tap11Base.metadata.create_all(engine) 

202 

203 for schema in normalized["@graph"]: 

204 tap_visitor = TapLoadingVisitor( 

205 engine, 

206 catalog_name=catalog_name, 

207 schema_name=schema_name, 

208 tap_tables=tap_tables, 

209 ) 

210 tap_visitor.visit_schema(schema) 

211 else: 

212 _insert_dump = InsertDump() 

213 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat") 

214 # After the engine is created, update the executor with the dialect 

215 _insert_dump.dialect = conn.dialect 

216 

217 for schema in normalized["@graph"]: 

218 tap_visitor = TapLoadingVisitor.from_mock_connection( 

219 conn, 

220 catalog_name=catalog_name, 

221 schema_name=schema_name, 

222 tap_tables=tap_tables, 

223 ) 

224 tap_visitor.visit_schema(schema) 

225 

226 

227@cli.command("modify-tap") 

228@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0) 

229@click.argument("files", nargs=-1, type=click.File()) 

230def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None: 

231 """Modify TAP information in Felis schema FILES. 

232 

233 This command has some utilities to aid in rewriting felis FILES 

234 in specific ways. It will write out a merged version of these files. 

235 """ 

236 count = 0 

237 graph = [] 

238 for file in files: 

239 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

240 if "@graph" not in schema_obj: 

241 schema_obj["@type"] = "felis:Schema" 

242 schema_obj["@context"] = DEFAULT_CONTEXT 

243 schema_index = schema_obj.get("tap:schema_index") 

244 if not schema_index or (schema_index and schema_index > start_schema_at): 

245 schema_index = start_schema_at + count 

246 count += 1 

247 schema_obj["tap:schema_index"] = schema_index 

248 graph.extend(jsonld.flatten(schema_obj)) 

249 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph} 

250 normalized = _normalize(merged, embed="@always") 

251 _dump(normalized) 

252 

253 

254@cli.command("basic-check") 

255@click.argument("file", type=click.File()) 

256def basic_check(file: io.TextIOBase) -> None: 

257 """Perform a basic check on a felis FILE. 

258 

259 This performs a very check to ensure required fields are 

260 populated and basic semantics are okay. It does not ensure semantics 

261 are valid for other commands like create-all or load-tap. 

262 """ 

263 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

264 schema_obj["@type"] = "felis:Schema" 

265 # Force Context and Schema Type 

266 schema_obj["@context"] = DEFAULT_CONTEXT 

267 check_visitor = CheckingVisitor() 

268 check_visitor.visit_schema(schema_obj) 

269 

270 

271@cli.command("normalize") 

272@click.argument("file", type=click.File()) 

273def normalize(file: io.TextIOBase) -> None: 

274 """Normalize a Felis FILE. 

275 

276 Takes a felis schema FILE, expands it (resolving the full URLs), 

277 then compacts it, and finally produces output in the canonical 

278 format. 

279 

280 (This is most useful in some debugging scenarios) 

281 

282 See Also : 

283 

284 https://json-ld.org/spec/latest/json-ld/#expanded-document-form 

285 https://json-ld.org/spec/latest/json-ld/#compacted-document-form 

286 """ 

287 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

288 schema_obj["@type"] = "felis:Schema" 

289 # Force Context and Schema Type 

290 schema_obj["@context"] = DEFAULT_CONTEXT 

291 expanded = jsonld.expand(schema_obj) 

292 normalized = _normalize(expanded, embed="@always") 

293 _dump(normalized) 

294 

295 

296@cli.command("merge") 

297@click.argument("files", nargs=-1, type=click.File()) 

298def merge(files: Iterable[io.TextIOBase]) -> None: 

299 """Merge a set of Felis FILES. 

300 

301 This will expand out the felis FILES so that it is easy to 

302 override values (using @Id), then normalize to a single 

303 output. 

304 """ 

305 graph = [] 

306 for file in files: 

307 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

308 if "@graph" not in schema_obj: 

309 schema_obj["@type"] = "felis:Schema" 

310 schema_obj["@context"] = DEFAULT_CONTEXT 

311 graph.extend(jsonld.flatten(schema_obj)) 

312 updated_map: MutableMapping[str, Any] = {} 

313 for item in graph: 

314 _id = item["@id"] 

315 item_to_update = updated_map.get(_id, item) 

316 if item_to_update and item_to_update != item: 

317 logger.debug(f"Overwriting {_id}") 

318 item_to_update.update(item) 

319 updated_map[_id] = item_to_update 

320 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())} 

321 normalized = _normalize(merged, embed="@always") 

322 _dump(normalized) 

323 

324 

325@cli.command("validate") 

326@click.option( 

327 "-s", 

328 "--schema-name", 

329 help="Schema name for validation", 

330 type=click.Choice(["RSP", "default"]), 

331 default="default", 

332) 

333@click.option("-d", "--require-description", is_flag=True, help="Require description for all objects") 

334@click.argument("files", nargs=-1, type=click.File()) 

335def validate(schema_name: str, require_description: bool, files: Iterable[io.TextIOBase]) -> None: 

336 """Validate one or more felis YAML files.""" 

337 schema_class = get_schema(schema_name) 

338 logger.info(f"Using schema '{schema_class.__name__}'") 

339 

340 if require_description: 

341 Schema.require_description(True) 

342 

343 rc = 0 

344 for file in files: 

345 file_name = getattr(file, "name", None) 

346 logger.info(f"Validating {file_name}") 

347 try: 

348 schema_class.model_validate(yaml.load(file, Loader=yaml.SafeLoader)) 

349 except ValidationError as e: 

350 logger.error(e) 

351 rc = 1 

352 if rc: 

353 raise click.exceptions.Exit(rc) 

354 

355 

356@cli.command("dump-json") 

357@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.") 

358@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.") 

359@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.") 

360@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.") 

361@click.argument("file", type=click.File()) 

362def dump_json( 

363 file: io.TextIOBase, 

364 expanded: bool = False, 

365 compacted: bool = False, 

366 framed: bool = False, 

367 graph: bool = False, 

368) -> None: 

369 """Dump JSON representation using various JSON-LD options.""" 

370 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

371 schema_obj["@type"] = "felis:Schema" 

372 # Force Context and Schema Type 

373 schema_obj["@context"] = DEFAULT_CONTEXT 

374 

375 if expanded: 

376 schema_obj = jsonld.expand(schema_obj) 

377 if framed: 

378 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME) 

379 if compacted: 

380 options = {} 

381 if graph: 

382 options["graph"] = True 

383 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options) 

384 json.dump(schema_obj, sys.stdout, indent=4) 

385 

386 

387def _dump(obj: Mapping[str, Any]) -> None: 

388 class OrderedDumper(yaml.Dumper): 

389 pass 

390 

391 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any: 

392 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) 

393 

394 OrderedDumper.add_representer(dict, _dict_representer) 

395 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False)) 

396 

397 

398def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]: 

399 framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed)) 

400 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True)) 

401 graph = compacted["@graph"] 

402 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph] 

403 compacted["@graph"] = graph if len(graph) > 1 else graph[0] 

404 return compacted 

405 

406 

407class InsertDump: 

408 """An Insert Dumper for SQL statements.""" 

409 

410 dialect: Any = None 

411 

412 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None: 

413 compiled = sql.compile(dialect=self.dialect) 

414 sql_str = str(compiled) + ";" 

415 params_list = [compiled.params] 

416 for params in params_list: 

417 if not params: 

418 print(sql_str) 

419 continue 

420 new_params = {} 

421 for key, value in params.items(): 

422 if isinstance(value, str): 

423 new_params[key] = f"'{value}'" 

424 elif value is None: 

425 new_params[key] = "null" 

426 else: 

427 new_params[key] = value 

428 

429 print(sql_str % new_params) 

430 

431 

432if __name__ == "__main__": 432 ↛ 433line 432 didn't jump to line 433, because the condition on line 432 was never true

433 cli()