Coverage for python/felis/cli.py: 54%

228 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-25 10:20 -0700

1# This file is part of felis. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import io 

25import json 

26import logging 

27import sys 

28from collections.abc import Iterable, Mapping, MutableMapping 

29from typing import IO, Any 

30 

31import click 

32import yaml 

33from pydantic import ValidationError 

34from pyld import jsonld 

35from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url 

36from sqlalchemy.engine.mock import MockConnection 

37 

38from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__ 

39from .check import CheckingVisitor 

40from .datamodel import Schema 

41from .metadata import DatabaseContext, InsertDump, MetaDataBuilder 

42from .tap import Tap11Base, TapLoadingVisitor, init_tables 

43from .utils import ReorderingVisitor 

44from .validation import get_schema 

45 

46logger = logging.getLogger("felis") 

47 

48loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"] 

49 

50 

51@click.group() 

52@click.version_option(__version__) 

53@click.option( 

54 "--log-level", 

55 type=click.Choice(loglevel_choices), 

56 envvar="FELIS_LOGLEVEL", 

57 help="Felis log level", 

58 default=logging.getLevelName(logging.INFO), 

59) 

60@click.option( 

61 "--log-file", 

62 type=click.Path(), 

63 envvar="FELIS_LOGFILE", 

64 help="Felis log file path", 

65) 

66def cli(log_level: str, log_file: str | None) -> None: 

67 """Felis Command Line Tools.""" 

68 if log_file: 

69 logging.basicConfig(filename=log_file, level=log_level) 

70 else: 

71 logging.basicConfig(level=log_level) 

72 

73 

74@cli.command("create") 

75@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://") 

76@click.option("--schema-name", help="Alternate schema name to override Felis file") 

77@click.option( 

78 "--create-if-not-exists", is_flag=True, help="Create the schema in the database if it does not exist" 

79) 

80@click.option("--drop-if-exists", is_flag=True, help="Drop schema if it already exists in the database") 

81@click.option("--echo", is_flag=True, help="Echo database commands as they are executed") 

82@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing") 

83@click.option( 

84 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing" 

85) 

86@click.argument("file", type=click.File()) 

87def create( 

88 engine_url: str, 

89 schema_name: str | None, 

90 create_if_not_exists: bool, 

91 drop_if_exists: bool, 

92 echo: bool, 

93 dry_run: bool, 

94 output_file: IO[str] | None, 

95 file: IO, 

96) -> None: 

97 """Create database objects from the Felis file.""" 

98 yaml_data = yaml.safe_load(file) 

99 schema = Schema.model_validate(yaml_data) 

100 url_obj = make_url(engine_url) 

101 if schema_name: 

102 logger.info(f"Overriding schema name with: {schema_name}") 

103 schema.name = schema_name 

104 elif url_obj.drivername == "sqlite": 

105 logger.info("Overriding schema name for sqlite with: main") 

106 schema.name = "main" 

107 if not url_obj.host and not url_obj.drivername == "sqlite": 

108 dry_run = True 

109 logger.info("Forcing dry run for non-sqlite engine URL with no host") 

110 

111 builder = MetaDataBuilder(schema) 

112 builder.build() 

113 metadata = builder.metadata 

114 logger.debug(f"Created metadata with schema name: {metadata.schema}") 

115 

116 engine: Engine | MockConnection 

117 if not dry_run and not output_file: 

118 engine = create_engine(engine_url, echo=echo) 

119 else: 

120 if dry_run: 

121 logger.info("Dry run will be executed") 

122 engine = DatabaseContext.create_mock_engine(url_obj, output_file) 

123 if output_file: 

124 logger.info("Writing SQL output to: " + output_file.name) 

125 

126 context = DatabaseContext(metadata, engine) 

127 

128 if drop_if_exists: 

129 logger.debug("Dropping schema if it exists") 

130 context.drop_if_exists() 

131 create_if_not_exists = True # If schema is dropped, it needs to be recreated. 

132 

133 if create_if_not_exists: 

134 logger.debug("Creating schema if not exists") 

135 context.create_if_not_exists() 

136 

137 context.create_all() 

138 

139 

140@cli.command("init-tap") 

141@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

142@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

143@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

144@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

145@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

146@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

147@click.argument("engine-url") 

148def init_tap( 

149 engine_url: str, 

150 tap_schema_name: str, 

151 tap_schemas_table: str, 

152 tap_tables_table: str, 

153 tap_columns_table: str, 

154 tap_keys_table: str, 

155 tap_key_columns_table: str, 

156) -> None: 

157 """Initialize TAP 1.1 TAP_SCHEMA objects. 

158 

159 Please verify the schema/catalog you are executing this in in your 

160 engine URL. 

161 """ 

162 engine = create_engine(engine_url, echo=True) 

163 init_tables( 

164 tap_schema_name, 

165 tap_schemas_table, 

166 tap_tables_table, 

167 tap_columns_table, 

168 tap_keys_table, 

169 tap_key_columns_table, 

170 ) 

171 Tap11Base.metadata.create_all(engine) 

172 

173 

174@cli.command("load-tap") 

175@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog") 

176@click.option("--schema-name", help="Alternate Schema Name for Felis file") 

177@click.option("--catalog-name", help="Catalog Name for Schema") 

178@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

179@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

180@click.option("--tap-tables-postfix", help="Postfix for TAP table names") 

181@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

182@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

183@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

184@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

185@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

186@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema") 

187@click.argument("file", type=click.File()) 

188def load_tap( 

189 engine_url: str, 

190 schema_name: str, 

191 catalog_name: str, 

192 dry_run: bool, 

193 tap_schema_name: str, 

194 tap_tables_postfix: str, 

195 tap_schemas_table: str, 

196 tap_tables_table: str, 

197 tap_columns_table: str, 

198 tap_keys_table: str, 

199 tap_key_columns_table: str, 

200 tap_schema_index: int, 

201 file: io.TextIOBase, 

202) -> None: 

203 """Load TAP metadata from a Felis FILE. 

204 

205 This command loads the associated TAP metadata from a Felis FILE 

206 to the TAP_SCHEMA tables. 

207 """ 

208 yaml_data = yaml.load(file, Loader=yaml.SafeLoader) 

209 schema = Schema.model_validate(yaml_data) 

210 

211 tap_tables = init_tables( 

212 tap_schema_name, 

213 tap_tables_postfix, 

214 tap_schemas_table, 

215 tap_tables_table, 

216 tap_columns_table, 

217 tap_keys_table, 

218 tap_key_columns_table, 

219 ) 

220 

221 if not dry_run: 

222 engine = create_engine(engine_url) 

223 

224 if engine_url == "sqlite://" and not dry_run: 

225 # In Memory SQLite - Mostly used to test 

226 Tap11Base.metadata.create_all(engine) 

227 

228 tap_visitor = TapLoadingVisitor( 

229 engine, 

230 catalog_name=catalog_name, 

231 schema_name=schema_name, 

232 tap_tables=tap_tables, 

233 tap_schema_index=tap_schema_index, 

234 ) 

235 tap_visitor.visit_schema(schema) 

236 else: 

237 _insert_dump = InsertDump() 

238 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat") 

239 # After the engine is created, update the executor with the dialect 

240 _insert_dump.dialect = conn.dialect 

241 

242 tap_visitor = TapLoadingVisitor.from_mock_connection( 

243 conn, 

244 catalog_name=catalog_name, 

245 schema_name=schema_name, 

246 tap_tables=tap_tables, 

247 tap_schema_index=tap_schema_index, 

248 ) 

249 tap_visitor.visit_schema(schema) 

250 

251 

252@cli.command("modify-tap") 

253@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0) 

254@click.argument("files", nargs=-1, type=click.File()) 

255def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None: 

256 """Modify TAP information in Felis schema FILES. 

257 

258 This command has some utilities to aid in rewriting felis FILES 

259 in specific ways. It will write out a merged version of these files. 

260 """ 

261 count = 0 

262 graph = [] 

263 for file in files: 

264 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

265 if "@graph" not in schema_obj: 

266 schema_obj["@type"] = "felis:Schema" 

267 schema_obj["@context"] = DEFAULT_CONTEXT 

268 schema_index = schema_obj.get("tap:schema_index") 

269 if not schema_index or (schema_index and schema_index > start_schema_at): 

270 schema_index = start_schema_at + count 

271 count += 1 

272 schema_obj["tap:schema_index"] = schema_index 

273 graph.extend(jsonld.flatten(schema_obj)) 

274 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph} 

275 normalized = _normalize(merged, embed="@always") 

276 _dump(normalized) 

277 

278 

279@cli.command("basic-check") 

280@click.argument("file", type=click.File()) 

281def basic_check(file: io.TextIOBase) -> None: 

282 """Perform a basic check on a felis FILE. 

283 

284 This performs a very check to ensure required fields are 

285 populated and basic semantics are okay. It does not ensure semantics 

286 are valid for other commands like create-all or load-tap. 

287 """ 

288 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

289 schema_obj["@type"] = "felis:Schema" 

290 # Force Context and Schema Type 

291 schema_obj["@context"] = DEFAULT_CONTEXT 

292 check_visitor = CheckingVisitor() 

293 check_visitor.visit_schema(schema_obj) 

294 

295 

296@cli.command("normalize") 

297@click.argument("file", type=click.File()) 

298def normalize(file: io.TextIOBase) -> None: 

299 """Normalize a Felis FILE. 

300 

301 Takes a felis schema FILE, expands it (resolving the full URLs), 

302 then compacts it, and finally produces output in the canonical 

303 format. 

304 

305 (This is most useful in some debugging scenarios) 

306 

307 See Also : 

308 

309 https://json-ld.org/spec/latest/json-ld/#expanded-document-form 

310 https://json-ld.org/spec/latest/json-ld/#compacted-document-form 

311 """ 

312 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

313 schema_obj["@type"] = "felis:Schema" 

314 # Force Context and Schema Type 

315 schema_obj["@context"] = DEFAULT_CONTEXT 

316 expanded = jsonld.expand(schema_obj) 

317 normalized = _normalize(expanded, embed="@always") 

318 _dump(normalized) 

319 

320 

321@cli.command("merge") 

322@click.argument("files", nargs=-1, type=click.File()) 

323def merge(files: Iterable[io.TextIOBase]) -> None: 

324 """Merge a set of Felis FILES. 

325 

326 This will expand out the felis FILES so that it is easy to 

327 override values (using @Id), then normalize to a single 

328 output. 

329 """ 

330 graph = [] 

331 for file in files: 

332 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

333 if "@graph" not in schema_obj: 

334 schema_obj["@type"] = "felis:Schema" 

335 schema_obj["@context"] = DEFAULT_CONTEXT 

336 graph.extend(jsonld.flatten(schema_obj)) 

337 updated_map: MutableMapping[str, Any] = {} 

338 for item in graph: 

339 _id = item["@id"] 

340 item_to_update = updated_map.get(_id, item) 

341 if item_to_update and item_to_update != item: 

342 logger.debug(f"Overwriting {_id}") 

343 item_to_update.update(item) 

344 updated_map[_id] = item_to_update 

345 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())} 

346 normalized = _normalize(merged, embed="@always") 

347 _dump(normalized) 

348 

349 

350@cli.command("validate") 

351@click.option( 

352 "-s", 

353 "--schema-name", 

354 help="Schema name for validation", 

355 type=click.Choice(["RSP", "default"]), 

356 default="default", 

357) 

358@click.option( 

359 "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False 

360) 

361@click.option( 

362 "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False 

363) 

364@click.argument("files", nargs=-1, type=click.File()) 

365def validate( 

366 schema_name: str, 

367 require_description: bool, 

368 check_redundant_datatypes: bool, 

369 files: Iterable[io.TextIOBase], 

370) -> None: 

371 """Validate one or more felis YAML files.""" 

372 schema_class = get_schema(schema_name) 

373 if schema_name != "default": 

374 logger.info(f"Using schema '{schema_class.__name__}'") 

375 

376 rc = 0 

377 for file in files: 

378 file_name = getattr(file, "name", None) 

379 logger.info(f"Validating {file_name}") 

380 try: 

381 data = yaml.load(file, Loader=yaml.SafeLoader) 

382 schema_class.model_validate( 

383 data, 

384 context={ 

385 "check_redundant_datatypes": check_redundant_datatypes, 

386 "require_description": require_description, 

387 }, 

388 ) 

389 except ValidationError as e: 

390 logger.error(e) 

391 rc = 1 

392 if rc: 

393 raise click.exceptions.Exit(rc) 

394 

395 

396@cli.command("dump-json") 

397@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.") 

398@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.") 

399@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.") 

400@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.") 

401@click.argument("file", type=click.File()) 

402def dump_json( 

403 file: io.TextIOBase, 

404 expanded: bool = False, 

405 compacted: bool = False, 

406 framed: bool = False, 

407 graph: bool = False, 

408) -> None: 

409 """Dump JSON representation using various JSON-LD options.""" 

410 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

411 schema_obj["@type"] = "felis:Schema" 

412 # Force Context and Schema Type 

413 schema_obj["@context"] = DEFAULT_CONTEXT 

414 

415 if expanded: 

416 schema_obj = jsonld.expand(schema_obj) 

417 if framed: 

418 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME) 

419 if compacted: 

420 options = {} 

421 if graph: 

422 options["graph"] = True 

423 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options) 

424 json.dump(schema_obj, sys.stdout, indent=4) 

425 

426 

427def _dump(obj: Mapping[str, Any]) -> None: 

428 class OrderedDumper(yaml.Dumper): 

429 pass 

430 

431 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any: 

432 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) 

433 

434 OrderedDumper.add_representer(dict, _dict_representer) 

435 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False)) 

436 

437 

438def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]: 

439 framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed)) 

440 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True)) 

441 graph = compacted["@graph"] 

442 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph] 

443 compacted["@graph"] = graph if len(graph) > 1 else graph[0] 

444 return compacted 

445 

446 

447if __name__ == "__main__": 447 ↛ 448line 447 didn't jump to line 448, because the condition on line 447 was never true

448 cli()