Coverage for python/felis/cli.py: 35%
176 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-07 01:49 -0800
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
32from sqlalchemy.engine import Engine, create_engine, make_url
33from sqlalchemy.engine.mock import MockConnection, create_mock_engine
35from . import __version__
36from .datamodel import Schema
37from .db.schema import create_database
38from .db.utils import DatabaseContext, is_mock_url
39from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
40from .metadata import MetaDataBuilder
41from .tap_schema import DataLoader, MetadataInserter, TableManager
43__all__ = ["cli"]
45logger = logging.getLogger("felis")
47loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
50@click.group()
51@click.version_option(__version__)
52@click.option(
53 "--log-level",
54 type=click.Choice(loglevel_choices),
55 envvar="FELIS_LOGLEVEL",
56 help="Felis log level",
57 default=logging.getLevelName(logging.INFO),
58)
59@click.option(
60 "--log-file",
61 type=click.Path(),
62 envvar="FELIS_LOGFILE",
63 help="Felis log file path",
64)
65@click.option(
66 "--id-generation/--no-id-generation",
67 is_flag=True,
68 help="Generate IDs for all objects that do not have them",
69 default=True,
70)
71@click.pass_context
72def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
73 """Felis command line tools"""
74 ctx.ensure_object(dict)
75 ctx.obj["id_generation"] = id_generation
76 if ctx.obj["id_generation"]:
77 logger.info("ID generation is enabled")
78 else:
79 logger.info("ID generation is disabled")
80 if log_file:
81 logging.basicConfig(filename=log_file, level=log_level)
82 else:
83 logging.basicConfig(level=log_level)
86@cli.command("create", help="Create database objects from the Felis file")
87@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
88@click.option("--schema-name", help="Alternate schema name to override Felis file")
89@click.option(
90 "--initialize",
91 is_flag=True,
92 help="Create the schema in the database if it does not exist (error if already exists)",
93)
94@click.option(
95 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
96)
97@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
98@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
99@click.option(
100 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
101)
102@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
103@click.argument("file", type=click.File())
104@click.pass_context
105def create(
106 ctx: click.Context,
107 engine_url: str,
108 schema_name: str | None,
109 initialize: bool,
110 drop: bool,
111 echo: bool,
112 dry_run: bool,
113 output_file: IO[str] | None,
114 ignore_constraints: bool,
115 file: IO[str],
116) -> None:
117 """Create database objects from the Felis file.
119 Parameters
120 ----------
121 engine_url
122 SQLAlchemy Engine URL.
123 schema_name
124 Alternate schema name to override Felis file.
125 initialize
126 Create the schema in the database if it does not exist.
127 drop
128 Drop schema if it already exists in the database.
129 echo
130 Echo database commands as they are executed.
131 dry_run
132 Dry run only to print out commands instead of executing.
133 output_file
134 Write SQL commands to a file instead of executing.
135 ignore_constraints
136 Ignore constraints when creating tables.
137 file
138 Felis file to read.
139 """
140 try:
141 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
142 url = make_url(engine_url)
143 if schema_name:
144 logger.info(f"Overriding schema name with: {schema_name}")
145 schema.name = schema_name
146 elif url.drivername == "sqlite":
147 logger.info("Overriding schema name for sqlite with: main")
148 schema.name = "main"
149 if not url.host and not url.drivername == "sqlite":
150 dry_run = True
151 logger.info("Forcing dry run for non-sqlite engine URL with no host")
153 metadata = MetaDataBuilder(schema, ignore_constraints=ignore_constraints).build()
154 logger.debug(f"Created metadata with schema name: {metadata.schema}")
156 engine: Engine | MockConnection
157 if not dry_run and not output_file:
158 engine = create_engine(url, echo=echo)
159 else:
160 if dry_run:
161 logger.info("Dry run will be executed")
162 engine = DatabaseContext.create_mock_engine(url, output_file)
163 if output_file:
164 logger.info("Writing SQL output to: " + output_file.name)
166 context = DatabaseContext(metadata, engine)
168 if drop and initialize:
169 raise ValueError("Cannot drop and initialize schema at the same time")
171 if drop:
172 logger.debug("Dropping schema if it exists")
173 context.drop()
174 initialize = True # If schema is dropped, it needs to be recreated.
176 if initialize:
177 logger.debug("Creating schema if not exists")
178 context.initialize()
180 context.create_all()
181 except Exception as e:
182 logger.exception(e)
183 raise click.ClickException(str(e))
186@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
187@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
188@click.option(
189 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
190)
191@click.option(
192 "--tap-tables-postfix",
193 "-p",
194 help="Postfix which is applied to standard TAP_SCHEMA table names",
195 default="",
196)
197@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
198@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
199@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
200@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
201@click.option(
202 "--force-unbounded-arraysize",
203 is_flag=True,
204 help="Use unbounded arraysize by default for all variable length string columns"
205 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
206) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
207@click.option(
208 "--unique-keys",
209 "-u",
210 is_flag=True,
211 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
212 default=False,
213)
214@click.argument("file", type=click.File())
215@click.pass_context
216def load_tap_schema(
217 ctx: click.Context,
218 engine_url: str,
219 tap_schema_name: str,
220 tap_tables_postfix: str,
221 tap_schema_index: int,
222 dry_run: bool,
223 echo: bool,
224 output_file: str | None,
225 force_unbounded_arraysize: bool,
226 unique_keys: bool,
227 file: IO[str],
228) -> None:
229 """Load TAP metadata from a Felis file.
231 Parameters
232 ----------
233 engine_url
234 SQLAlchemy Engine URL.
235 tap_tables_postfix
236 Postfix which is applied to standard TAP_SCHEMA table names.
237 tap_schema_index
238 TAP_SCHEMA index of the schema in this environment.
239 dry_run
240 Execute dry run only. Does not insert any data.
241 echo
242 Print out the generated insert statements to stdout.
243 output_file
244 Output file for writing generated SQL.
245 file
246 Felis file to read.
248 Notes
249 -----
250 The TAP_SCHEMA database must already exist or the command will fail. This
251 command will not initialize the TAP_SCHEMA tables.
252 """
253 url = make_url(engine_url)
254 engine: Engine | MockConnection
255 if dry_run or is_mock_url(url):
256 engine = create_mock_engine(url, executor=None)
257 else:
258 engine = create_engine(engine_url)
259 mgr = TableManager(
260 engine=engine,
261 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
262 schema_name=tap_schema_name,
263 table_name_postfix=tap_tables_postfix,
264 )
266 schema = Schema.from_stream(
267 file,
268 context={
269 "id_generation": ctx.obj["id_generation"],
270 "force_unbounded_arraysize": force_unbounded_arraysize,
271 },
272 )
274 DataLoader(
275 schema,
276 mgr,
277 engine,
278 tap_schema_index=tap_schema_index,
279 dry_run=dry_run,
280 print_sql=echo,
281 output_path=output_file,
282 unique_keys=unique_keys,
283 ).load()
286@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
287@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
288@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
289@click.option(
290 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
291)
292@click.option(
293 "--insert-metadata/--no-insert-metadata",
294 is_flag=True,
295 help="Insert metadata describing TAP_SCHEMA itself",
296 default=True,
297)
298@click.pass_context
299def init_tap_schema(
300 ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
301) -> None:
302 """Initialize a standard TAP_SCHEMA database.
304 Parameters
305 ----------
306 engine_url
307 SQLAlchemy Engine URL.
308 tap_schema_name
309 Name of the TAP_SCHEMA schema in the database.
310 tap_tables_postfix
311 Postfix which is applied to standard TAP_SCHEMA table names.
312 insert_metadata
313 Insert metadata describing TAP_SCHEMA itself.
314 If set to False, only the TAP_SCHEMA tables will be created, but no
315 metadata will be inserted.
316 """
317 url = make_url(engine_url)
318 engine: Engine | MockConnection
319 if is_mock_url(url):
320 raise click.ClickException("Mock engine URL is not supported for this command")
321 engine = create_engine(engine_url)
322 mgr = TableManager(
323 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
324 schema_name=tap_schema_name,
325 table_name_postfix=tap_tables_postfix,
326 )
327 mgr.initialize_database(engine)
328 if insert_metadata:
329 inserter = MetadataInserter(mgr, engine)
330 inserter.insert_metadata()
333@cli.command("validate", help="Validate one or more Felis YAML files")
334@click.option(
335 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
336)
337@click.option(
338 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
339)
340@click.option(
341 "--check-tap-table-indexes",
342 is_flag=True,
343 help="Check that every table has a unique TAP table index",
344 default=False,
345)
346@click.option(
347 "--check-tap-principal",
348 is_flag=True,
349 help="Check that at least one column per table is flagged as TAP principal",
350 default=False,
351)
352@click.argument("files", nargs=-1, type=click.File())
353@click.pass_context
354def validate(
355 ctx: click.Context,
356 check_description: bool,
357 check_redundant_datatypes: bool,
358 check_tap_table_indexes: bool,
359 check_tap_principal: bool,
360 files: Iterable[IO[str]],
361) -> None:
362 """Validate one or more felis YAML files.
364 Parameters
365 ----------
366 check_description
367 Check that all objects have a valid description.
368 check_redundant_datatypes
369 Check for redundant type overrides.
370 check_tap_table_indexes
371 Check that every table has a unique TAP table index.
372 check_tap_principal
373 Check that at least one column per table is flagged as TAP principal.
374 files
375 The Felis YAML files to validate.
377 Raises
378 ------
379 click.exceptions.Exit
380 Raised if any validation errors are found. The ``ValidationError``
381 which is thrown when a schema fails to validate will be logged as an
382 error message.
384 Notes
385 -----
386 All of the ``check`` flags are turned off by default and represent
387 optional validations controlled by the Pydantic context.
388 """
389 rc = 0
390 for file in files:
391 file_name = getattr(file, "name", None)
392 logger.info(f"Validating {file_name}")
393 try:
394 Schema.from_stream(
395 file,
396 context={
397 "check_description": check_description,
398 "check_redundant_datatypes": check_redundant_datatypes,
399 "check_tap_table_indexes": check_tap_table_indexes,
400 "check_tap_principal": check_tap_principal,
401 "id_generation": ctx.obj["id_generation"],
402 },
403 )
404 except ValidationError as e:
405 logger.error(e)
406 rc = 1
407 if rc:
408 raise click.exceptions.Exit(rc)
411@cli.command(
412 "diff",
413 help="""
414 Compare two schemas or a schema and a database for changes
416 Examples:
418 felis diff schema1.yaml schema2.yaml
420 felis diff -c alembic schema1.yaml schema2.yaml
422 felis diff --engine-url sqlite:///test.db schema.yaml
423 """,
424)
425@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
426@click.option(
427 "-c",
428 "--comparator",
429 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
430 help="Comparator to use for schema comparison",
431 default="deepdiff",
432)
433@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
434@click.argument("files", nargs=-1, type=click.File())
435@click.pass_context
436def diff(
437 ctx: click.Context,
438 engine_url: str | None,
439 comparator: str,
440 error_on_change: bool,
441 files: Iterable[IO[str]],
442) -> None:
443 schemas = [
444 Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
445 ]
447 diff: SchemaDiff
448 if len(schemas) == 2 and engine_url is None:
449 if comparator == "alembic":
450 db_context = create_database(schemas[0])
451 assert isinstance(db_context.engine, Engine)
452 diff = DatabaseDiff(schemas[1], db_context.engine)
453 else:
454 diff = FormattedSchemaDiff(schemas[0], schemas[1])
455 elif len(schemas) == 1 and engine_url is not None:
456 engine = create_engine(engine_url)
457 diff = DatabaseDiff(schemas[0], engine)
458 else:
459 raise click.ClickException(
460 "Invalid arguments - provide two schemas or a schema and a database engine URL"
461 )
463 diff.print()
465 if diff.has_changes and error_on_change:
466 raise click.ClickException("Schema was changed")
469@cli.command(
470 "dump",
471 help="""
472 Dump a schema file to YAML or JSON format
474 Example:
476 felis dump schema.yaml schema.json
478 felis dump schema.yaml schema_dump.yaml
479 """,
480)
481@click.option(
482 "--strip-ids/--no-strip-ids",
483 is_flag=True,
484 help="Strip IDs from the output schema",
485 default=False,
486)
487@click.argument("files", nargs=2, type=click.Path())
488@click.pass_context
489def dump(
490 ctx: click.Context,
491 strip_ids: bool,
492 files: list[str],
493) -> None:
494 if strip_ids:
495 logger.info("Stripping IDs from the output schema")
496 if files[1].endswith(".json"):
497 format = "json"
498 elif files[1].endswith(".yaml"):
499 format = "yaml"
500 else:
501 raise click.ClickException("Output file must have a .json or .yaml extension")
502 schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
503 with open(files[1], "w") as f:
504 if format == "yaml":
505 schema.dump_yaml(f, strip_ids=strip_ids)
506 elif format == "json":
507 schema.dump_json(f, strip_ids=strip_ids)
510if __name__ == "__main__": 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true
511 cli()