Coverage for python/felis/cli.py: 35%
215 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-24 01:22 -0800
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
32from sqlalchemy.engine import Engine, create_engine, make_url
33from sqlalchemy.engine.mock import MockConnection, create_mock_engine
35from . import __version__
36from .datamodel import Schema
37from .db.schema import create_database
38from .db.utils import DatabaseContext, is_mock_url
39from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
40from .metadata import MetaDataBuilder
41from .tap_schema import DataLoader, MetadataInserter, TableManager
43__all__ = ["cli"]
45logger = logging.getLogger("felis")
47loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
50def _create_database_context(
51 schema: Schema,
52 engine_url: str,
53 schema_name: str | None = None,
54) -> DatabaseContext:
55 if schema_name:
56 logger.info(f"Overriding schema name with: {schema_name}")
57 schema.name = schema_name
58 metadata = MetaDataBuilder(
59 schema,
60 ).build()
61 url = make_url(engine_url)
62 engine = create_engine(url)
63 if engine.dialect.name == "sqlite":
64 schema.name = "main"
65 logger.info("Setting schema name to 'main' for SQLite")
66 return DatabaseContext(metadata, engine)
69@click.group()
70@click.version_option(__version__)
71@click.option(
72 "--log-level",
73 type=click.Choice(loglevel_choices),
74 envvar="FELIS_LOGLEVEL",
75 help="Felis log level",
76 default=logging.getLevelName(logging.INFO),
77)
78@click.option(
79 "--log-file",
80 type=click.Path(),
81 envvar="FELIS_LOGFILE",
82 help="Felis log file path",
83)
84@click.option(
85 "--id-generation/--no-id-generation",
86 is_flag=True,
87 help="Generate IDs for all objects that do not have them",
88 default=True,
89)
90@click.pass_context
91def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
92 """Felis command line tools"""
93 ctx.ensure_object(dict)
94 ctx.obj["id_generation"] = id_generation
95 if ctx.obj["id_generation"]:
96 logger.info("ID generation is enabled")
97 else:
98 logger.info("ID generation is disabled")
99 if log_file:
100 logging.basicConfig(filename=log_file, level=log_level)
101 else:
102 logging.basicConfig(level=log_level)
105@cli.command("create", help="Create database objects from the Felis file")
106@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
107@click.option("--schema-name", help="Alternate schema name to override Felis file")
108@click.option(
109 "--initialize",
110 is_flag=True,
111 help="Create the schema in the database if it does not exist (error if already exists)",
112)
113@click.option(
114 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
115)
116@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
117@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
118@click.option(
119 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
120)
121@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
122@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
123@click.argument("file", type=click.File())
124@click.pass_context
125def create(
126 ctx: click.Context,
127 engine_url: str,
128 schema_name: str | None,
129 initialize: bool,
130 drop: bool,
131 echo: bool,
132 dry_run: bool,
133 output_file: IO[str] | None,
134 ignore_constraints: bool,
135 skip_indexes: bool,
136 file: IO[str],
137) -> None:
138 """Create database objects from the Felis file.
140 Parameters
141 ----------
142 engine_url
143 SQLAlchemy Engine URL.
144 schema_name
145 Alternate schema name to override Felis file.
146 initialize
147 Create the schema in the database if it does not exist.
148 drop
149 Drop schema if it already exists in the database.
150 echo
151 Echo database commands as they are executed.
152 dry_run
153 Dry run only to print out commands instead of executing.
154 output_file
155 Write SQL commands to a file instead of executing.
156 ignore_constraints
157 Ignore constraints when creating tables.
158 skip_indexes
159 Skip creating indexes when building metadata.
160 file
161 Felis file to read.
162 """
163 try:
164 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
165 url = make_url(engine_url)
166 if schema_name:
167 logger.info(f"Overriding schema name with: {schema_name}")
168 schema.name = schema_name
169 elif url.drivername == "sqlite":
170 logger.info("Overriding schema name for sqlite with: main")
171 schema.name = "main"
172 if not url.host and not url.drivername == "sqlite":
173 dry_run = True
174 logger.info("Forcing dry run for non-sqlite engine URL with no host")
176 metadata = MetaDataBuilder(
177 schema,
178 ignore_constraints=ignore_constraints,
179 skip_indexes=skip_indexes,
180 ).build()
181 logger.debug(f"Created metadata with schema name: {metadata.schema}")
183 engine: Engine | MockConnection
184 if not dry_run and not output_file:
185 engine = create_engine(url, echo=echo)
186 else:
187 if dry_run:
188 logger.info("Dry run will be executed")
189 engine = DatabaseContext.create_mock_engine(url, output_file)
190 if output_file:
191 logger.info("Writing SQL output to: " + output_file.name)
193 context = DatabaseContext(metadata, engine)
195 if drop and initialize:
196 raise ValueError("Cannot drop and initialize schema at the same time")
198 if drop:
199 logger.debug("Dropping schema if it exists")
200 context.drop()
201 initialize = True # If schema is dropped, it needs to be recreated.
203 if initialize:
204 logger.debug("Creating schema if not exists")
205 context.initialize()
207 context.create_all()
208 except Exception as e:
209 logger.exception(e)
210 raise click.ClickException(str(e))
213@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
214@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
215@click.option("--schema-name", help="Alternate schema name to override Felis file")
216@click.argument("file", type=click.File())
217@click.pass_context
218def create_indexes(
219 ctx: click.Context,
220 engine_url: str,
221 schema_name: str | None,
222 file: IO[str],
223) -> None:
224 """Create indexes from a Felis YAML file in a target database.
226 Parameters
227 ----------
228 engine_url
229 SQLAlchemy Engine URL.
230 file
231 Felis file to read.
232 """
233 try:
234 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
235 db = _create_database_context(schema, engine_url, schema_name)
236 db.create_indexes()
237 except Exception as e:
238 logger.exception(e)
239 raise click.ClickException("Error creating indexes: " + str(e))
242@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
243@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
244@click.option("--schema-name", help="Alternate schema name to override Felis file")
245@click.argument("file", type=click.File())
246@click.pass_context
247def drop_indexes(
248 ctx: click.Context,
249 engine_url: str,
250 schema_name: str | None,
251 file: IO[str],
252) -> None:
253 """Drop indexes from a Felis YAML file in a target database.
255 Parameters
256 ----------
257 engine_url
258 SQLAlchemy Engine URL.
259 schema-name
260 Alternate schema name to override Felis file.
261 file
262 Felis file to read.
263 """
264 try:
265 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
266 db = _create_database_context(schema, engine_url, schema_name)
267 db.drop_indexes()
268 except Exception as e:
269 logger.exception(e)
270 raise click.ClickException("Error dropping indexes: " + str(e))
273@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
274@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
275@click.option(
276 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
277)
278@click.option(
279 "--tap-tables-postfix",
280 "-p",
281 help="Postfix which is applied to standard TAP_SCHEMA table names",
282 default="",
283)
284@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
285@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
286@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
287@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
288@click.option(
289 "--force-unbounded-arraysize",
290 is_flag=True,
291 help="Use unbounded arraysize by default for all variable length string columns"
292 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
293) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
294@click.option(
295 "--unique-keys",
296 "-u",
297 is_flag=True,
298 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
299 default=False,
300)
301@click.argument("file", type=click.File())
302@click.pass_context
303def load_tap_schema(
304 ctx: click.Context,
305 engine_url: str,
306 tap_schema_name: str,
307 tap_tables_postfix: str,
308 tap_schema_index: int,
309 dry_run: bool,
310 echo: bool,
311 output_file: str | None,
312 force_unbounded_arraysize: bool,
313 unique_keys: bool,
314 file: IO[str],
315) -> None:
316 """Load TAP metadata from a Felis file.
318 Parameters
319 ----------
320 engine_url
321 SQLAlchemy Engine URL.
322 tap_tables_postfix
323 Postfix which is applied to standard TAP_SCHEMA table names.
324 tap_schema_index
325 TAP_SCHEMA index of the schema in this environment.
326 dry_run
327 Execute dry run only. Does not insert any data.
328 echo
329 Print out the generated insert statements to stdout.
330 output_file
331 Output file for writing generated SQL.
332 file
333 Felis file to read.
335 Notes
336 -----
337 The TAP_SCHEMA database must already exist or the command will fail. This
338 command will not initialize the TAP_SCHEMA tables.
339 """
340 url = make_url(engine_url)
341 engine: Engine | MockConnection
342 if dry_run or is_mock_url(url):
343 engine = create_mock_engine(url, executor=None)
344 else:
345 engine = create_engine(engine_url)
346 mgr = TableManager(
347 engine=engine,
348 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
349 schema_name=tap_schema_name,
350 table_name_postfix=tap_tables_postfix,
351 )
353 schema = Schema.from_stream(
354 file,
355 context={
356 "id_generation": ctx.obj["id_generation"],
357 "force_unbounded_arraysize": force_unbounded_arraysize,
358 },
359 )
361 DataLoader(
362 schema,
363 mgr,
364 engine,
365 tap_schema_index=tap_schema_index,
366 dry_run=dry_run,
367 print_sql=echo,
368 output_path=output_file,
369 unique_keys=unique_keys,
370 ).load()
373@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
374@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
375@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
376@click.option(
377 "--extensions",
378 type=str,
379 default=None,
380 help=(
381 "Optional path to extensions YAML file (system path or resource:// URI). "
382 "If not provided, no extensions will be applied. "
383 "Example (default packaged extensions): "
384 "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
385 ),
386)
387@click.option(
388 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
389)
390@click.option(
391 "--insert-metadata/--no-insert-metadata",
392 is_flag=True,
393 help="Insert metadata describing TAP_SCHEMA itself",
394 default=True,
395)
396@click.pass_context
397def init_tap_schema(
398 ctx: click.Context,
399 engine_url: str,
400 tap_schema_name: str,
401 extensions: str | None,
402 tap_tables_postfix: str,
403 insert_metadata: bool,
404) -> None:
405 """Initialize a standard TAP_SCHEMA database.
407 Parameters
408 ----------
409 engine_url
410 SQLAlchemy Engine URL.
411 tap_schema_name
412 Name of the TAP_SCHEMA schema in the database.
413 extensions
414 Extensions YAML file.
415 tap_tables_postfix
416 Postfix which is applied to standard TAP_SCHEMA table names.
417 insert_metadata
418 Insert metadata describing TAP_SCHEMA itself.
419 If set to False, only the TAP_SCHEMA tables will be created, but no
420 metadata will be inserted.
421 """
422 url = make_url(engine_url)
423 engine: Engine | MockConnection
424 if is_mock_url(url):
425 raise click.ClickException("Mock engine URL is not supported for this command")
426 engine = create_engine(engine_url)
427 mgr = TableManager(
428 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
429 schema_name=tap_schema_name,
430 table_name_postfix=tap_tables_postfix,
431 extensions_path=extensions,
432 )
433 mgr.initialize_database(engine)
434 if insert_metadata:
435 inserter = MetadataInserter(mgr, engine)
436 inserter.insert_metadata()
439@cli.command("validate", help="Validate one or more Felis YAML files")
440@click.option(
441 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
442)
443@click.option(
444 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
445)
446@click.option(
447 "--check-tap-table-indexes",
448 is_flag=True,
449 help="Check that every table has a unique TAP table index",
450 default=False,
451)
452@click.option(
453 "--check-tap-principal",
454 is_flag=True,
455 help="Check that at least one column per table is flagged as TAP principal",
456 default=False,
457)
458@click.argument("files", nargs=-1, type=click.File())
459@click.pass_context
460def validate(
461 ctx: click.Context,
462 check_description: bool,
463 check_redundant_datatypes: bool,
464 check_tap_table_indexes: bool,
465 check_tap_principal: bool,
466 files: Iterable[IO[str]],
467) -> None:
468 """Validate one or more felis YAML files.
470 Parameters
471 ----------
472 check_description
473 Check that all objects have a valid description.
474 check_redundant_datatypes
475 Check for redundant type overrides.
476 check_tap_table_indexes
477 Check that every table has a unique TAP table index.
478 check_tap_principal
479 Check that at least one column per table is flagged as TAP principal.
480 files
481 The Felis YAML files to validate.
483 Raises
484 ------
485 click.exceptions.Exit
486 Raised if any validation errors are found. The ``ValidationError``
487 which is thrown when a schema fails to validate will be logged as an
488 error message.
490 Notes
491 -----
492 All of the ``check`` flags are turned off by default and represent
493 optional validations controlled by the Pydantic context.
494 """
495 rc = 0
496 for file in files:
497 file_name = getattr(file, "name", None)
498 logger.info(f"Validating {file_name}")
499 try:
500 Schema.from_stream(
501 file,
502 context={
503 "check_description": check_description,
504 "check_redundant_datatypes": check_redundant_datatypes,
505 "check_tap_table_indexes": check_tap_table_indexes,
506 "check_tap_principal": check_tap_principal,
507 "id_generation": ctx.obj["id_generation"],
508 },
509 )
510 except ValidationError as e:
511 logger.error(e)
512 rc = 1
513 if rc:
514 raise click.exceptions.Exit(rc)
517@cli.command(
518 "diff",
519 help="""
520 Compare two schemas or a schema and a database for changes
522 Examples:
524 felis diff schema1.yaml schema2.yaml
526 felis diff -c alembic schema1.yaml schema2.yaml
528 felis diff --engine-url sqlite:///test.db schema.yaml
529 """,
530)
531@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
532@click.option(
533 "-c",
534 "--comparator",
535 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
536 help="Comparator to use for schema comparison",
537 default="deepdiff",
538)
539@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
540@click.argument("files", nargs=-1, type=click.File())
541@click.pass_context
542def diff(
543 ctx: click.Context,
544 engine_url: str | None,
545 comparator: str,
546 error_on_change: bool,
547 files: Iterable[IO[str]],
548) -> None:
549 schemas = [
550 Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
551 ]
553 diff: SchemaDiff
554 if len(schemas) == 2 and engine_url is None:
555 if comparator == "alembic":
556 db_context = create_database(schemas[0])
557 assert isinstance(db_context.engine, Engine)
558 diff = DatabaseDiff(schemas[1], db_context.engine)
559 else:
560 diff = FormattedSchemaDiff(schemas[0], schemas[1])
561 elif len(schemas) == 1 and engine_url is not None:
562 engine = create_engine(engine_url)
563 diff = DatabaseDiff(schemas[0], engine)
564 else:
565 raise click.ClickException(
566 "Invalid arguments - provide two schemas or a schema and a database engine URL"
567 )
569 diff.print()
571 if diff.has_changes and error_on_change:
572 raise click.ClickException("Schema was changed")
575@cli.command(
576 "dump",
577 help="""
578 Dump a schema file to YAML or JSON format
580 Example:
582 felis dump schema.yaml schema.json
584 felis dump schema.yaml schema_dump.yaml
585 """,
586)
587@click.option(
588 "--strip-ids/--no-strip-ids",
589 is_flag=True,
590 help="Strip IDs from the output schema",
591 default=False,
592)
593@click.argument("files", nargs=2, type=click.Path())
594@click.pass_context
595def dump(
596 ctx: click.Context,
597 strip_ids: bool,
598 files: list[str],
599) -> None:
600 if strip_ids:
601 logger.info("Stripping IDs from the output schema")
602 if files[1].endswith(".json"):
603 format = "json"
604 elif files[1].endswith(".yaml"):
605 format = "yaml"
606 else:
607 raise click.ClickException("Output file must have a .json or .yaml extension")
608 schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
609 with open(files[1], "w") as f:
610 if format == "yaml":
611 schema.dump_yaml(f, strip_ids=strip_ids)
612 elif format == "json":
613 schema.dump_json(f, strip_ids=strip_ids)
616if __name__ == "__main__": 616 ↛ 617line 616 didn't jump to line 617 because the condition on line 616 was never true
617 cli()