Coverage for python/felis/cli.py: 61%

131 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-30 02:49 -0700

1# This file is part of felis. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import io 

25import logging 

26from collections.abc import Iterable 

27from typing import IO 

28 

29import click 

30import yaml 

31from pydantic import ValidationError 

32from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url 

33from sqlalchemy.engine.mock import MockConnection 

34 

35from . import __version__ 

36from .datamodel import Schema 

37from .metadata import DatabaseContext, InsertDump, MetaDataBuilder 

38from .tap import Tap11Base, TapLoadingVisitor, init_tables 

39from .validation import get_schema 

40 

41logger = logging.getLogger("felis") 

42 

43loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"] 

44 

45 

46@click.group() 

47@click.version_option(__version__) 

48@click.option( 

49 "--log-level", 

50 type=click.Choice(loglevel_choices), 

51 envvar="FELIS_LOGLEVEL", 

52 help="Felis log level", 

53 default=logging.getLevelName(logging.INFO), 

54) 

55@click.option( 

56 "--log-file", 

57 type=click.Path(), 

58 envvar="FELIS_LOGFILE", 

59 help="Felis log file path", 

60) 

61def cli(log_level: str, log_file: str | None) -> None: 

62 """Felis Command Line Tools.""" 

63 if log_file: 

64 logging.basicConfig(filename=log_file, level=log_level) 

65 else: 

66 logging.basicConfig(level=log_level) 

67 

68 

69@cli.command("create") 

70@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://") 

71@click.option("--schema-name", help="Alternate schema name to override Felis file") 

72@click.option( 

73 "--create-if-not-exists", is_flag=True, help="Create the schema in the database if it does not exist" 

74) 

75@click.option("--drop-if-exists", is_flag=True, help="Drop schema if it already exists in the database") 

76@click.option("--echo", is_flag=True, help="Echo database commands as they are executed") 

77@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing") 

78@click.option( 

79 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing" 

80) 

81@click.argument("file", type=click.File()) 

82def create( 

83 engine_url: str, 

84 schema_name: str | None, 

85 create_if_not_exists: bool, 

86 drop_if_exists: bool, 

87 echo: bool, 

88 dry_run: bool, 

89 output_file: IO[str] | None, 

90 file: IO, 

91) -> None: 

92 """Create database objects from the Felis file.""" 

93 yaml_data = yaml.safe_load(file) 

94 schema = Schema.model_validate(yaml_data) 

95 url_obj = make_url(engine_url) 

96 if schema_name: 

97 logger.info(f"Overriding schema name with: {schema_name}") 

98 schema.name = schema_name 

99 elif url_obj.drivername == "sqlite": 

100 logger.info("Overriding schema name for sqlite with: main") 

101 schema.name = "main" 

102 if not url_obj.host and not url_obj.drivername == "sqlite": 

103 dry_run = True 

104 logger.info("Forcing dry run for non-sqlite engine URL with no host") 

105 

106 builder = MetaDataBuilder(schema) 

107 builder.build() 

108 metadata = builder.metadata 

109 logger.debug(f"Created metadata with schema name: {metadata.schema}") 

110 

111 engine: Engine | MockConnection 

112 if not dry_run and not output_file: 

113 engine = create_engine(engine_url, echo=echo) 

114 else: 

115 if dry_run: 

116 logger.info("Dry run will be executed") 

117 engine = DatabaseContext.create_mock_engine(url_obj, output_file) 

118 if output_file: 

119 logger.info("Writing SQL output to: " + output_file.name) 

120 

121 context = DatabaseContext(metadata, engine) 

122 

123 if drop_if_exists: 

124 logger.debug("Dropping schema if it exists") 

125 context.drop_if_exists() 

126 create_if_not_exists = True # If schema is dropped, it needs to be recreated. 

127 

128 if create_if_not_exists: 

129 logger.debug("Creating schema if not exists") 

130 context.create_if_not_exists() 

131 

132 context.create_all() 

133 

134 

135@cli.command("init-tap") 

136@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

137@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

138@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

139@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

140@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

141@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

142@click.argument("engine-url") 

143def init_tap( 

144 engine_url: str, 

145 tap_schema_name: str, 

146 tap_schemas_table: str, 

147 tap_tables_table: str, 

148 tap_columns_table: str, 

149 tap_keys_table: str, 

150 tap_key_columns_table: str, 

151) -> None: 

152 """Initialize TAP 1.1 TAP_SCHEMA objects. 

153 

154 Please verify the schema/catalog you are executing this in in your 

155 engine URL. 

156 """ 

157 engine = create_engine(engine_url, echo=True) 

158 init_tables( 

159 tap_schema_name, 

160 tap_schemas_table, 

161 tap_tables_table, 

162 tap_columns_table, 

163 tap_keys_table, 

164 tap_key_columns_table, 

165 ) 

166 Tap11Base.metadata.create_all(engine) 

167 

168 

169@cli.command("load-tap") 

170@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog") 

171@click.option("--schema-name", help="Alternate Schema Name for Felis file") 

172@click.option("--catalog-name", help="Catalog Name for Schema") 

173@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed") 

174@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA") 

175@click.option("--tap-tables-postfix", help="Postfix for TAP table names") 

176@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas") 

177@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables") 

178@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns") 

179@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys") 

180@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns") 

181@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema") 

182@click.argument("file", type=click.File()) 

183def load_tap( 

184 engine_url: str, 

185 schema_name: str, 

186 catalog_name: str, 

187 dry_run: bool, 

188 tap_schema_name: str, 

189 tap_tables_postfix: str, 

190 tap_schemas_table: str, 

191 tap_tables_table: str, 

192 tap_columns_table: str, 

193 tap_keys_table: str, 

194 tap_key_columns_table: str, 

195 tap_schema_index: int, 

196 file: io.TextIOBase, 

197) -> None: 

198 """Load TAP metadata from a Felis FILE. 

199 

200 This command loads the associated TAP metadata from a Felis FILE 

201 to the TAP_SCHEMA tables. 

202 """ 

203 yaml_data = yaml.load(file, Loader=yaml.SafeLoader) 

204 schema = Schema.model_validate(yaml_data) 

205 

206 tap_tables = init_tables( 

207 tap_schema_name, 

208 tap_tables_postfix, 

209 tap_schemas_table, 

210 tap_tables_table, 

211 tap_columns_table, 

212 tap_keys_table, 

213 tap_key_columns_table, 

214 ) 

215 

216 if not dry_run: 

217 engine = create_engine(engine_url) 

218 

219 if engine_url == "sqlite://" and not dry_run: 

220 # In Memory SQLite - Mostly used to test 

221 Tap11Base.metadata.create_all(engine) 

222 

223 tap_visitor = TapLoadingVisitor( 

224 engine, 

225 catalog_name=catalog_name, 

226 schema_name=schema_name, 

227 tap_tables=tap_tables, 

228 tap_schema_index=tap_schema_index, 

229 ) 

230 tap_visitor.visit_schema(schema) 

231 else: 

232 _insert_dump = InsertDump() 

233 conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat") 

234 # After the engine is created, update the executor with the dialect 

235 _insert_dump.dialect = conn.dialect 

236 

237 tap_visitor = TapLoadingVisitor.from_mock_connection( 

238 conn, 

239 catalog_name=catalog_name, 

240 schema_name=schema_name, 

241 tap_tables=tap_tables, 

242 tap_schema_index=tap_schema_index, 

243 ) 

244 tap_visitor.visit_schema(schema) 

245 

246 

247@cli.command("validate") 

248@click.option( 

249 "-s", 

250 "--schema-name", 

251 help="Schema name for validation", 

252 type=click.Choice(["RSP", "default"]), 

253 default="default", 

254) 

255@click.option( 

256 "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False 

257) 

258@click.option( 

259 "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False 

260) 

261@click.argument("files", nargs=-1, type=click.File()) 

262def validate( 

263 schema_name: str, 

264 require_description: bool, 

265 check_redundant_datatypes: bool, 

266 files: Iterable[io.TextIOBase], 

267) -> None: 

268 """Validate one or more felis YAML files.""" 

269 schema_class = get_schema(schema_name) 

270 if schema_name != "default": 

271 logger.info(f"Using schema '{schema_class.__name__}'") 

272 

273 rc = 0 

274 for file in files: 

275 file_name = getattr(file, "name", None) 

276 logger.info(f"Validating {file_name}") 

277 try: 

278 data = yaml.load(file, Loader=yaml.SafeLoader) 

279 schema_class.model_validate( 

280 data, 

281 context={ 

282 "check_redundant_datatypes": check_redundant_datatypes, 

283 "require_description": require_description, 

284 }, 

285 ) 

286 except ValidationError as e: 

287 logger.error(e) 

288 rc = 1 

289 if rc: 

290 raise click.exceptions.Exit(rc) 

291 

292 

293if __name__ == "__main__": 293 ↛ 294line 293 didn't jump to line 294, because the condition on line 293 was never true

294 cli()