Coverage for python/felis/cli.py: 29%

201 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-22 01:55 -0700

1# This file is part of felis. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import io 

23import json 

24import logging 

25import sys 

26from collections.abc import Iterable, Mapping, MutableMapping 

27from typing import Any 

28 

29import click 

30import yaml 

31from pyld import jsonld 

32from sqlalchemy import create_engine 

33 

34from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__ 

35from .check import CheckingVisitor 

36from .sql import SQLVisitor 

37from .tap import Tap11Base, TapLoadingVisitor, init_tables 

38from .utils import ReorderingVisitor 

39 

40logger = logging.getLogger("felis") 

41 

42 

43@click.group() 

44@click.version_option(__version__) 

45def cli() -> None: 

46 """Felis Command Line Tools""" 

47 logging.basicConfig(level=logging.INFO) 

48 

49 

50@cli.command("create-all") 

51@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL") 

52@click.option("--schema-name", help="Alternate Schema Name for Felis File") 

53@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

54@click.argument("file", type=click.File()) 

55def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None: 

56 """Create schema objects from the Felis FILE.""" 

57 

58 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

59 visitor = SQLVisitor(schema_name=schema_name) 

60 schema = visitor.visit_schema(schema_obj) 

61 

62 metadata = schema.metadata 

63 

64 if not dry_run: 

65 engine = create_engine(engine_url) 

66 else: 

67 _insert_dump = InsertDump() 

68 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump) 

69 _insert_dump.dialect = engine.dialect 

70 metadata.create_all(engine) 

71 

72 

73@cli.command("init-tap") 

74@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

75@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

76@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

77@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

78@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

79@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

80@click.argument("engine-url") 

81def init_tap( 

82 engine_url: str, 

83 tap_schema_name: str, 

84 tap_schemas_table: str, 

85 tap_tables_table: str, 

86 tap_columns_table: str, 

87 tap_keys_table: str, 

88 tap_key_columns_table: str, 

89) -> None: 

90 """Initialize TAP 1.1 TAP_SCHEMA objects. 

91 Please verify the schema/catalog you are executing this in in your 

92 engine URL.""" 

93 engine = create_engine(engine_url, echo=True) 

94 init_tables( 

95 tap_schema_name, 

96 tap_schemas_table, 

97 tap_tables_table, 

98 tap_columns_table, 

99 tap_keys_table, 

100 tap_key_columns_table, 

101 ) 

102 Tap11Base.metadata.create_all(engine) 

103 

104 

105@cli.command("load-tap") 

106@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog") 

107@click.option("--schema-name", help="Alternate Schema Name for Felis file") 

108@click.option("--catalog-name", help="Catalog Name for Schema") 

109@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

110@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

111@click.option("--tap-tables-postfix", help="Postfix for TAP table names") 

112@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

113@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

114@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

115@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

116@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

117@click.argument("file", type=click.File()) 

118def load_tap( 

119 engine_url: str, 

120 schema_name: str, 

121 catalog_name: str, 

122 dry_run: bool, 

123 tap_schema_name: str, 

124 tap_tables_postfix: str, 

125 tap_schemas_table: str, 

126 tap_tables_table: str, 

127 tap_columns_table: str, 

128 tap_keys_table: str, 

129 tap_key_columns_table: str, 

130 file: io.TextIOBase, 

131) -> None: 

132 """Load TAP metadata from a Felis FILE. 

133 This command loads the associated TAP metadata from a Felis FILE 

134 to the TAP_SCHEMA tables.""" 

135 top_level_object = yaml.load(file, Loader=yaml.SafeLoader) 

136 schema_obj: dict 

137 if isinstance(top_level_object, dict): 

138 schema_obj = top_level_object 

139 if "@graph" not in schema_obj: 

140 schema_obj["@type"] = "felis:Schema" 

141 schema_obj["@context"] = DEFAULT_CONTEXT 

142 elif isinstance(top_level_object, list): 

143 schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object} 

144 else: 

145 logger.error("Schema object not of recognizable type") 

146 sys.exit(1) 

147 

148 normalized = _normalize(schema_obj) 

149 if len(normalized["@graph"]) > 1 and (schema_name or catalog_name): 

150 logger.error("--schema-name and --catalog-name incompatible with multiple schemas") 

151 sys.exit(1) 

152 

153 # Force normalized["@graph"] to a list, which is what happens when there's 

154 # multiple schemas 

155 if isinstance(normalized["@graph"], dict): 

156 normalized["@graph"] = [normalized["@graph"]] 

157 

158 if not dry_run: 

159 engine = create_engine(engine_url) 

160 else: 

161 _insert_dump = InsertDump() 

162 engine = create_engine(engine_url, strategy="mock", executor=_insert_dump.dump, paramstyle="pyformat") 

163 # After the engine is created, update the executor with the dialect 

164 _insert_dump.dialect = engine.dialect 

165 tap_tables = init_tables( 

166 tap_schema_name, 

167 tap_tables_postfix, 

168 tap_schemas_table, 

169 tap_tables_table, 

170 tap_columns_table, 

171 tap_keys_table, 

172 tap_key_columns_table, 

173 ) 

174 

175 if engine_url == "sqlite://" and not dry_run: 

176 # In Memory SQLite - Mostly used to test 

177 Tap11Base.metadata.create_all(engine) 

178 

179 for schema in normalized["@graph"]: 

180 tap_visitor = TapLoadingVisitor( 

181 engine, catalog_name=catalog_name, schema_name=schema_name, mock=dry_run, tap_tables=tap_tables 

182 ) 

183 tap_visitor.visit_schema(schema) 

184 

185 

186@cli.command("modify-tap") 

187@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index") 

188@click.argument("files", nargs=-1, type=click.File()) 

189def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None: 

190 """Modify TAP information in Felis schema FILES. 

191 This command has some utilities to aid in rewriting felis FILES 

192 in specific ways. It will write out a merged version of these files. 

193 """ 

194 count = 0 

195 graph = [] 

196 for file in files: 

197 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

198 if "@graph" not in schema_obj: 

199 schema_obj["@type"] = "felis:Schema" 

200 schema_obj["@context"] = DEFAULT_CONTEXT 

201 schema_index = schema_obj.get("tap:schema_index") 

202 if not schema_index or (schema_index and schema_index > start_schema_at): 

203 schema_index = start_schema_at + count 

204 count += 1 

205 schema_obj["tap:schema_index"] = schema_index 

206 graph.extend(jsonld.flatten(schema_obj)) 

207 merged = {"@context": DEFAULT_CONTEXT, "@graph": graph} 

208 normalized = _normalize(merged) 

209 _dump(normalized) 

210 

211 

212@cli.command("basic-check") 

213@click.argument("file", type=click.File()) 

214def basic_check(file: io.TextIOBase) -> None: 

215 """Perform a basic check on a felis FILE. 

216 This performs a very check to ensure required fields are 

217 populated and basic semantics are okay. It does not ensure semantics 

218 are valid for other commands like create-all or load-tap. 

219 """ 

220 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

221 schema_obj["@type"] = "felis:Schema" 

222 # Force Context and Schema Type 

223 schema_obj["@context"] = DEFAULT_CONTEXT 

224 check_visitor = CheckingVisitor() 

225 check_visitor.visit_schema(schema_obj) 

226 

227 

228@cli.command("normalize") 

229@click.argument("file", type=click.File()) 

230def normalize(file: io.TextIOBase) -> None: 

231 """Normalize a Felis FILE. 

232 Takes a felis schema FILE, expands it (resolving the full URLs), 

233 then compacts it, and finally produces output in the canonical 

234 format. 

235 

236 (This is most useful in some debugging scenarios) 

237 

238 See Also: 

239 https://json-ld.org/spec/latest/json-ld/#expanded-document-form 

240 https://json-ld.org/spec/latest/json-ld/#compacted-document-form 

241 """ 

242 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

243 schema_obj["@type"] = "felis:Schema" 

244 # Force Context and Schema Type 

245 schema_obj["@context"] = DEFAULT_CONTEXT 

246 expanded = jsonld.expand(schema_obj) 

247 normalized = _normalize(expanded) 

248 _dump(normalized) 

249 

250 

251@cli.command("merge") 

252@click.argument("files", nargs=-1, type=click.File()) 

253def merge(files: Iterable[io.TextIOBase]) -> None: 

254 """Merge a set of Felis FILES. 

255 

256 This will expand out the felis FILES so that it is easy to 

257 override values (using @Id), then normalize to a single 

258 output. 

259 """ 

260 graph = [] 

261 for file in files: 

262 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

263 if "@graph" not in schema_obj: 

264 schema_obj["@type"] = "felis:Schema" 

265 schema_obj["@context"] = DEFAULT_CONTEXT 

266 graph.extend(jsonld.flatten(schema_obj)) 

267 updated_map: MutableMapping[str, Any] = {} 

268 for item in graph: 

269 _id = item["@id"] 

270 item_to_update = updated_map.get(_id, item) 

271 if item_to_update and item_to_update != item: 

272 logger.debug(f"Overwriting {_id}") 

273 item_to_update.update(item) 

274 updated_map[_id] = item_to_update 

275 merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())} 

276 normalized = _normalize(merged) 

277 _dump(normalized) 

278 

279 

280@cli.command("dump-json") 

281@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.") 

282@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.") 

283@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.") 

284@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.") 

285@click.argument("file", type=click.File()) 

286def dump_json( 

287 file: io.TextIOBase, 

288 expanded: bool = False, 

289 compacted: bool = False, 

290 framed: bool = False, 

291 graph: bool = False, 

292) -> None: 

293 """Dump JSON representation using various JSON-LD options.""" 

294 schema_obj = yaml.load(file, Loader=yaml.SafeLoader) 

295 schema_obj["@type"] = "felis:Schema" 

296 # Force Context and Schema Type 

297 schema_obj["@context"] = DEFAULT_CONTEXT 

298 

299 if expanded: 

300 schema_obj = jsonld.expand(schema_obj) 

301 if framed: 

302 schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME) 

303 if compacted: 

304 options = {} 

305 if graph: 

306 options["graph"] = True 

307 schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options) 

308 json.dump(schema_obj, sys.stdout, indent=4) 

309 

310 

311def _dump(obj: Mapping[str, Any]) -> None: 

312 class OrderedDumper(yaml.Dumper): 

313 pass 

314 

315 def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any: 

316 return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) 

317 

318 OrderedDumper.add_representer(dict, _dict_representer) 

319 print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False)) 

320 

321 

322def _normalize(schema_obj: Mapping[str, Any]) -> MutableMapping[str, Any]: 

323 framed = jsonld.frame(schema_obj, DEFAULT_FRAME) 

324 compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True)) 

325 graph = compacted["@graph"] 

326 graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph] 

327 compacted["@graph"] = graph if len(graph) > 1 else graph[0] 

328 return compacted 

329 

330 

331class InsertDump: 

332 """An Insert Dumper for SQL statements""" 

333 

334 dialect: Any = None 

335 

336 def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None: 

337 compiled = sql.compile(dialect=self.dialect) 

338 sql_str = str(compiled) + ";" 

339 params_list = [compiled.params] 

340 for params in params_list: 

341 if not params: 

342 print(sql_str) 

343 continue 

344 new_params = {} 

345 for key, value in params.items(): 

346 if isinstance(value, str): 

347 new_params[key] = f"'{value}'" 

348 elif value is None: 

349 new_params[key] = "null" 

350 else: 

351 new_params[key] = value 

352 

353 print(sql_str % new_params) 

354 

355 

356if __name__ == "__main__": 356 ↛ 357line 356 didn't jump to line 357, because the condition on line 356 was never true

357 cli()