Coverage for python/felis/cli.py: 30%

202 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-09 03:01 -0800

1# This file is part of felis. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import io 

23import json 

24import logging 

25import sys 

26from collections.abc import Iterable, Mapping, MutableMapping 

27from typing import Any 

28 

29import click 

30import yaml 

31from pyld import jsonld 

32from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url 

33from sqlalchemy.engine.mock import MockConnection 

34 

35from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__ 

36from .check import CheckingVisitor 

37from .sql import SQLVisitor 

38from .tap import Tap11Base, TapLoadingVisitor, init_tables 

39from .utils import ReorderingVisitor 

40 

41logger = logging.getLogger("felis") 

42 

43 

44@click.group() 

45@click.version_option(__version__) 

46def cli() -> None: 

47 """Felis Command Line Tools""" 

48 logging.basicConfig(level=logging.INFO) 

49 

50 

51@cli.command("create-all") 

52@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL") 

53@click.option("--schema-name", help="Alternate Schema Name for Felis File") 

54@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

55@click.argument("file", type=click.File()) 

56def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None: 

57 """Create schema objects from the Felis FILE.""" 

58 

59 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

60 visitor = SQLVisitor(schema_name=schema_name) 

61 schema = visitor.visit_schema(schema_obj) 

62 

63 metadata = schema.metadata 

64 

65 engine: Engine | MockConnection 

66 if not dry_run: 

67 engine = create_engine(engine_url) 

68 else: 

69 _insert_dump = InsertDump() 

70 engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump) 

71 _insert_dump.dialect = engine.dialect 

72 metadata.create_all(engine) 

73 

74 

75@cli.command("init-tap") 

76@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

77@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

78@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

79@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

80@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

81@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

82@click.argument("engine-url") 

83def init_tap( 

84 engine_url: str, 

85 tap_schema_name: str, 

86 tap_schemas_table: str, 

87 tap_tables_table: str, 

88 tap_columns_table: str, 

89 tap_keys_table: str, 

90 tap_key_columns_table: str, 

91) -> None: 

92 """Initialize TAP 1.1 TAP_SCHEMA objects. 

93 Please verify the schema/catalog you are executing this in in your 

94 engine URL.""" 

95 engine = create_engine(engine_url, echo=True) 

96 init_tables( 

97 tap_schema_name, 

98 tap_schemas_table, 

99 tap_tables_table, 

100 tap_columns_table, 

101 tap_keys_table, 

102 tap_key_columns_table, 

103 ) 

104 Tap11Base.metadata.create_all(engine) 

105 

106 

107@cli.command("load-tap") 

108@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog") 

109@click.option("--schema-name", help="Alternate Schema Name for Felis file") 

110@click.option("--catalog-name", help="Catalog Name for Schema") 

111@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

112@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

113@click.option("--tap-tables-postfix", help="Postfix for TAP table names") 

114@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

115@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

116@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

117@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

118@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

119@click.argument("file", type=click.File()) 

120def load_tap( 

121 engine_url: str, 

122 schema_name: str, 

123 catalog_name: str, 

124 dry_run: bool, 

125 tap_schema_name: str, 

126 tap_tables_postfix: str, 

127 tap_schemas_table: str, 

128 tap_tables_table: str, 

129 tap_columns_table: str, 

130 tap_keys_table: str, 

131 tap_key_columns_table: str, 

132 file: io.TextIOBase, 

133) -> None: 

134 """Load TAP metadata from a Felis FILE. 

135 This command loads the associated TAP metadata from a Felis FILE 

136 to the TAP_SCHEMA tables.""" 

137 top_level_object = yaml.load(file, Loader=yaml.SafeLoader) 

138 schema_obj: dict 

139 if isinstance(top_level_object, dict): 

140 schema_obj = top_level_object 

141 if "@graph" not in schema_obj: 

142 schema_obj["@type"] = "felis:Schema" 

143 schema_obj["@context"] = DEFAULT_CONTEXT 

144 elif isinstance(top_level_object, list): 

145 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object} 

146 else: 

147 logger.error("Schema object not of recognizable type") 

148 sys.exit(1) 

149 

150 normalized = _normalize(schema_obj) 

151 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name): 

152 logger.error("--schema-name and --catalog-name incompatible with multiple schemas") 

153 sys.exit(1) 

154 

155 # Force normalized["@graph"] to a list, which is what happens when there's 

156 # multiple schemas 

157 if isinstance(normalized["@graph"], dict): 

158 normalized["@graph"] = [normalized["@graph"]] 

159 

160 if not dry_run: 

161 engine = create_engine(engine_url) 

162 else: 

163 _insert_dump = InsertDump() 

164 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump, paramstyle="pyformat") 

165 # After the engine is created, update the executor with the dialect 

166 _insert_dump.dialect = engine.dialect 

167 tap_tables = init_tables( 

168 tap_schema_name, 

169 tap_tables_postfix, 

170 tap_schemas_table, 

171 tap_tables_table, 

172 tap_columns_table, 

173 tap_keys_table, 

174 tap_key_columns_table, 

175 ) 

176 

177 if engine_url == "sqlite://" and not dry_run: 

178 # In Memory SQLite - Mostly used to test 

179 Tap11Base.metadata.create_all(engine) 

180 

181 for schema in normalized["@graph"]: 

182 tap_visitor = TapLoadingVisitor( 

183 engine, catalog_name=catalog_name, schema_name=schema_name, mock=dry_run, tap_tables=tap_tables 

184 ) 

185 tap_visitor.visit_schema(schema) 

186 

187 

188@cli.command("modify-tap") 

189@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index") 

190@click.argument("files", nargs=-1, type=click.File()) 

191def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None: 

192 """Modify TAP information in Felis schema FILES. 

193 This command has some utilities to aid in rewriting felis FILES 

194 in specific ways. It will write out a merged version of these files. 

195 """ 

196 count = 0 

197 graph = [] 

198 for file in files: 

199 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

200 if "@graph" not in schema_obj: 

201 schema_obj["@type"] = "felis:Schema" 

202 schema_obj["@context"] = DEFAULT_CONTEXT 

203 schema_index = schema_obj.get("tap:schema_index") 

204 if not schema_index or (schema_index and schema_index > start_schema_at): 

205 schema_index = start_schema_at + count 

206 count += 1 

207 schema_obj["tap:schema_index"] = schema_index 

208 graph.extend(jsonld.flatten(schema_obj)) 

209 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph} 

210 normalized = _normalize(merged) 

211 _dump(normalized) 

212 

213 

214@cli.command("basic-check") 

215@click.argument("file", type=click.File()) 

216def basic_check(file: io.TextIOBase) -> None: 

217 """Perform a basic check on a felis FILE. 

218 This performs a very check to ensure required fields are 

219 populated and basic semantics are okay. It does not ensure semantics 

220 are valid for other commands like create-all or load-tap. 

221 """ 

222 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

223 schema_obj["@type"] = "felis:Schema" 

224 # Force Context and Schema Type 

225 schema_obj["@context"] = DEFAULT_CONTEXT 

226 check_visitor = CheckingVisitor() 

227 check_visitor.visit_schema(schema_obj) 

228 

229 

230@cli.command("normalize") 

231@click.argument("file", type=click.File()) 

232def normalize(file: io.TextIOBase) -> None: 

233 """Normalize a Felis FILE. 

234 Takes a felis schema FILE, expands it (resolving the full URLs), 

235 then compacts it, and finally produces output in the canonical 

236 format. 

237 

238 (This is most useful in some debugging scenarios) 

239 

240 See Also: 

241 https://json-ld.org/spec/latest/json-ld/#expanded-document-form 

242 https://json-ld.org/spec/latest/json-ld/#compacted-document-form 

243 """ 

244 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

245 schema_obj["@type"] = "felis:Schema" 

246 # Force Context and Schema Type 

247 schema_obj["@context"] = DEFAULT_CONTEXT 

248 expanded = jsonld.expand(schema_obj) 

249 normalized = _normalize(expanded) 

250 _dump(normalized) 

251 

252 

253@cli.command("merge") 

254@click.argument("files", nargs=-1, type=click.File()) 

255def merge(files: Iterable[io.TextIOBase]) -> None: 

256 """Merge a set of Felis FILES. 

257 

258 This will expand out the felis FILES so that it is easy to 

259 override values (using @Id), then normalize to a single 

260 output. 

261 """ 

262 graph = [] 

263 for file in files: 

264 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

265 if "@graph" not in schema_obj: 

266 schema_obj["@type"] = "felis:Schema" 

267 schema_obj["@context"] = DEFAULT_CONTEXT 

268 graph.extend(jsonld.flatten(schema_obj)) 

269 updated_map: MutableMapping[str, Any] = {} 

270 for item in graph: 

271 _id = item["@id"] 

272 item_to_update = updated_map.get(_id, item) 

273 if item_to_update and item_to_update != item: 

274 logger.debug(f"Overwriting {_id}") 

275 item_to_update.update(item) 

276 updated_map[_id] = item_to_update 

277 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())} 

278 normalized = _normalize(merged) 

279 _dump(normalized) 

280 

281 

282@cli.command("dump-json") 

283@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.") 

284@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.") 

285@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.") 

286@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.") 

287@click.argument("file", type=click.File()) 

288def dump_json( 

289 file: io.TextIOBase, 

290 expanded: bool = False, 

291 compacted: bool = False, 

292 framed: bool = False, 

293 graph: bool = False, 

294) -> None: 

295 """Dump JSON representation using various JSON-LD options.""" 

296 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

297 schema_obj["@type"] = "felis:Schema" 

298 # Force Context and Schema Type 

299 schema_obj["@context"] = DEFAULT_CONTEXT 

300 

301 if expanded: 

302 schema_obj = jsonld.expand(schema_obj) 

303 if framed: 

304 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME) 

305 if compacted: 

306 options = {} 

307 if graph: 

308 options["graph"] = True 

309 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options) 

310 json.dump(schema_obj, sys.stdout, indent=4) 

311 

312 

313def _dump(obj: Mapping[str, Any]) -> None: 

314 class OrderedDumper(yaml.Dumper): 

315 pass 

316 

317 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any: 

318 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) 

319 

320 OrderedDumper.add_representer(dict, _dict_representer) 

321 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False)) 

322 

323 

324def _normalize(schema_obj: Mapping[str, Any]) -> MutableMapping[str, Any]: 

325 framed = jsonld.frame(schema_obj, DEFAULT_FRAME) 

326 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True)) 

327 graph = compacted["@graph"] 

328 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph] 

329 compacted["@graph"] = graph if len(graph) > 1 else graph[0] 

330 return compacted 

331 

332 

333class InsertDump: 

334 """An Insert Dumper for SQL statements""" 

335 

336 dialect: Any = None 

337 

338 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None: 

339 compiled = sql.compile(dialect=self.dialect) 

340 sql_str = str(compiled) + ";" 

341 params_list = [compiled.params] 

342 for params in params_list: 

343 if not params: 

344 print(sql_str) 

345 continue 

346 new_params = {} 

347 for key, value in params.items(): 

348 if isinstance(value, str): 

349 new_params[key] = f"'{value}'" 

350 elif value is None: 

351 new_params[key] = "null" 

352 else: 

353 new_params[key] = value 

354 

355 print(sql_str % new_params) 

356 

357 

358if __name__ == "__main__": 358 ↛ 359line 358 didn't jump to line 359, because the condition on line 358 was never true

359 cli()