Coverage for python/felis/cli.py: 34%
214 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-19 01:33 -0800
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
32from sqlalchemy.engine import Engine, create_engine, make_url
33from sqlalchemy.engine.mock import MockConnection, create_mock_engine
35from . import __version__
36from .datamodel import Schema
37from .db.schema import create_database
38from .db.utils import DatabaseContext, is_mock_url
39from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
40from .metadata import MetaDataBuilder
41from .tap_schema import DataLoader, MetadataInserter, TableManager
43__all__ = ["cli"]
45logger = logging.getLogger("felis")
47loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
50def _create_database_context(
51 schema: Schema,
52 engine_url: str,
53 schema_name: str | None = None,
54) -> DatabaseContext:
55 if schema_name:
56 logger.info(f"Overriding schema name with: {schema_name}")
57 schema.name = schema_name
58 metadata = MetaDataBuilder(
59 schema,
60 ).build()
61 url = make_url(engine_url)
62 engine = create_engine(url)
63 if engine.dialect.name == "sqlite":
64 schema.name = "main"
65 logger.info("Setting schema name to 'main' for SQLite")
66 return DatabaseContext(metadata, engine)
69@click.group()
70@click.version_option(__version__)
71@click.option(
72 "--log-level",
73 type=click.Choice(loglevel_choices),
74 envvar="FELIS_LOGLEVEL",
75 help="Felis log level",
76 default=logging.getLevelName(logging.INFO),
77)
78@click.option(
79 "--log-file",
80 type=click.Path(),
81 envvar="FELIS_LOGFILE",
82 help="Felis log file path",
83)
84@click.option(
85 "--id-generation/--no-id-generation",
86 is_flag=True,
87 help="Generate IDs for all objects that do not have them",
88 default=True,
89)
90@click.pass_context
91def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
92 """Felis command line tools"""
93 ctx.ensure_object(dict)
94 ctx.obj["id_generation"] = id_generation
95 if ctx.obj["id_generation"]:
96 logger.info("ID generation is enabled")
97 else:
98 logger.info("ID generation is disabled")
99 if log_file:
100 logging.basicConfig(filename=log_file, level=log_level)
101 else:
102 logging.basicConfig(level=log_level)
105@cli.command("create", help="Create database objects from the Felis file")
106@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
107@click.option("--schema-name", help="Alternate schema name to override Felis file")
108@click.option(
109 "--initialize",
110 is_flag=True,
111 help="Create the schema in the database if it does not exist (error if already exists)",
112)
113@click.option(
114 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
115)
116@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
117@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
118@click.option(
119 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
120)
121@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
122@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
123@click.argument("file", type=click.File())
124@click.pass_context
125def create(
126 ctx: click.Context,
127 engine_url: str,
128 schema_name: str | None,
129 initialize: bool,
130 drop: bool,
131 echo: bool,
132 dry_run: bool,
133 output_file: IO[str] | None,
134 ignore_constraints: bool,
135 skip_indexes: bool,
136 file: IO[str],
137) -> None:
138 """Create database objects from the Felis file.
140 Parameters
141 ----------
142 engine_url
143 SQLAlchemy Engine URL.
144 schema_name
145 Alternate schema name to override Felis file.
146 initialize
147 Create the schema in the database if it does not exist.
148 drop
149 Drop schema if it already exists in the database.
150 echo
151 Echo database commands as they are executed.
152 dry_run
153 Dry run only to print out commands instead of executing.
154 output_file
155 Write SQL commands to a file instead of executing.
156 ignore_constraints
157 Ignore constraints when creating tables.
158 skip_indexes
159 Skip creating indexes when building metadata.
160 file
161 Felis file to read.
162 """
163 try:
164 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
165 url = make_url(engine_url)
166 if schema_name:
167 logger.info(f"Overriding schema name with: {schema_name}")
168 schema.name = schema_name
169 elif url.drivername == "sqlite":
170 logger.info("Overriding schema name for sqlite with: main")
171 schema.name = "main"
172 if not url.host and not url.drivername == "sqlite":
173 dry_run = True
174 logger.info("Forcing dry run for non-sqlite engine URL with no host")
176 metadata = MetaDataBuilder(
177 schema,
178 ignore_constraints=ignore_constraints,
179 skip_indexes=skip_indexes,
180 ).build()
181 logger.debug(f"Created metadata with schema name: {metadata.schema}")
183 engine: Engine | MockConnection
184 if not dry_run and not output_file:
185 engine = create_engine(url, echo=echo)
186 else:
187 if dry_run:
188 logger.info("Dry run will be executed")
189 engine = DatabaseContext.create_mock_engine(url, output_file)
190 if output_file:
191 logger.info("Writing SQL output to: " + output_file.name)
193 context = DatabaseContext(metadata, engine)
195 if drop and initialize:
196 raise ValueError("Cannot drop and initialize schema at the same time")
198 if drop:
199 logger.debug("Dropping schema if it exists")
200 context.drop()
201 initialize = True # If schema is dropped, it needs to be recreated.
203 if initialize:
204 logger.debug("Creating schema if not exists")
205 context.initialize()
207 context.create_all()
208 except Exception as e:
209 logger.exception(e)
210 raise click.ClickException(str(e))
213@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
214@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
215@click.option("--schema-name", help="Alternate schema name to override Felis file")
216@click.argument("file", type=click.File())
217@click.pass_context
218def create_indexes(
219 ctx: click.Context,
220 engine_url: str,
221 schema_name: str | None,
222 file: IO[str],
223) -> None:
224 """Create indexes from a Felis YAML file in a target database.
226 Parameters
227 ----------
228 engine_url
229 SQLAlchemy Engine URL.
230 file
231 Felis file to read.
232 """
233 try:
234 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
235 db = _create_database_context(schema, engine_url, schema_name)
236 db.create_indexes()
237 except Exception as e:
238 logger.exception(e)
239 raise click.ClickException("Error creating indexes: " + str(e))
242@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
243@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
244@click.option("--schema-name", help="Alternate schema name to override Felis file")
245@click.argument("file", type=click.File())
246@click.pass_context
247def drop_indexes(
248 ctx: click.Context,
249 engine_url: str,
250 schema_name: str | None,
251 file: IO[str],
252) -> None:
253 """Drop indexes from a Felis YAML file in a target database.
255 Parameters
256 ----------
257 engine_url
258 SQLAlchemy Engine URL.
259 schema-name
260 Alternate schema name to override Felis file.
261 file
262 Felis file to read.
263 """
264 try:
265 schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
266 db = _create_database_context(schema, engine_url, schema_name)
267 db.drop_indexes()
268 except Exception as e:
269 logger.exception(e)
270 raise click.ClickException("Error dropping indexes: " + str(e))
273@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
274@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
275@click.option(
276 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
277)
278@click.option(
279 "--tap-tables-postfix",
280 "-p",
281 help="Postfix which is applied to standard TAP_SCHEMA table names",
282 default="",
283)
284@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
285@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
286@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
287@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
288@click.option(
289 "--force-unbounded-arraysize",
290 is_flag=True,
291 help="Use unbounded arraysize by default for all variable length string columns"
292 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
293) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
294@click.option(
295 "--unique-keys",
296 "-u",
297 is_flag=True,
298 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
299 default=False,
300)
301@click.argument("file", type=click.File())
302@click.pass_context
303def load_tap_schema(
304 ctx: click.Context,
305 engine_url: str,
306 tap_schema_name: str,
307 tap_tables_postfix: str,
308 tap_schema_index: int,
309 dry_run: bool,
310 echo: bool,
311 output_file: str | None,
312 force_unbounded_arraysize: bool,
313 unique_keys: bool,
314 file: IO[str],
315) -> None:
316 """Load TAP metadata from a Felis file.
318 Parameters
319 ----------
320 engine_url
321 SQLAlchemy Engine URL.
322 tap_tables_postfix
323 Postfix which is applied to standard TAP_SCHEMA table names.
324 tap_schema_index
325 TAP_SCHEMA index of the schema in this environment.
326 dry_run
327 Execute dry run only. Does not insert any data.
328 echo
329 Print out the generated insert statements to stdout.
330 output_file
331 Output file for writing generated SQL.
332 file
333 Felis file to read.
335 Notes
336 -----
337 The TAP_SCHEMA database must already exist or the command will fail. This
338 command will not initialize the TAP_SCHEMA tables.
339 """
340 url = make_url(engine_url)
341 engine: Engine | MockConnection
342 if dry_run or is_mock_url(url):
343 engine = create_mock_engine(url, executor=None)
344 else:
345 engine = create_engine(engine_url)
346 mgr = TableManager(
347 engine=engine,
348 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
349 schema_name=tap_schema_name,
350 table_name_postfix=tap_tables_postfix,
351 )
353 schema = Schema.from_stream(
354 file,
355 context={
356 "id_generation": ctx.obj["id_generation"],
357 "force_unbounded_arraysize": force_unbounded_arraysize,
358 },
359 )
361 DataLoader(
362 schema,
363 mgr,
364 engine,
365 tap_schema_index=tap_schema_index,
366 dry_run=dry_run,
367 print_sql=echo,
368 output_path=output_file,
369 unique_keys=unique_keys,
370 ).load()
373@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
374@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
375@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
376@click.option(
377 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
378)
379@click.option(
380 "--insert-metadata/--no-insert-metadata",
381 is_flag=True,
382 help="Insert metadata describing TAP_SCHEMA itself",
383 default=True,
384)
385@click.pass_context
386def init_tap_schema(
387 ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
388) -> None:
389 """Initialize a standard TAP_SCHEMA database.
391 Parameters
392 ----------
393 engine_url
394 SQLAlchemy Engine URL.
395 tap_schema_name
396 Name of the TAP_SCHEMA schema in the database.
397 tap_tables_postfix
398 Postfix which is applied to standard TAP_SCHEMA table names.
399 insert_metadata
400 Insert metadata describing TAP_SCHEMA itself.
401 If set to False, only the TAP_SCHEMA tables will be created, but no
402 metadata will be inserted.
403 """
404 url = make_url(engine_url)
405 engine: Engine | MockConnection
406 if is_mock_url(url):
407 raise click.ClickException("Mock engine URL is not supported for this command")
408 engine = create_engine(engine_url)
409 mgr = TableManager(
410 apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
411 schema_name=tap_schema_name,
412 table_name_postfix=tap_tables_postfix,
413 )
414 mgr.initialize_database(engine)
415 if insert_metadata:
416 inserter = MetadataInserter(mgr, engine)
417 inserter.insert_metadata()
420@cli.command("validate", help="Validate one or more Felis YAML files")
421@click.option(
422 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
423)
424@click.option(
425 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
426)
427@click.option(
428 "--check-tap-table-indexes",
429 is_flag=True,
430 help="Check that every table has a unique TAP table index",
431 default=False,
432)
433@click.option(
434 "--check-tap-principal",
435 is_flag=True,
436 help="Check that at least one column per table is flagged as TAP principal",
437 default=False,
438)
439@click.argument("files", nargs=-1, type=click.File())
440@click.pass_context
441def validate(
442 ctx: click.Context,
443 check_description: bool,
444 check_redundant_datatypes: bool,
445 check_tap_table_indexes: bool,
446 check_tap_principal: bool,
447 files: Iterable[IO[str]],
448) -> None:
449 """Validate one or more felis YAML files.
451 Parameters
452 ----------
453 check_description
454 Check that all objects have a valid description.
455 check_redundant_datatypes
456 Check for redundant type overrides.
457 check_tap_table_indexes
458 Check that every table has a unique TAP table index.
459 check_tap_principal
460 Check that at least one column per table is flagged as TAP principal.
461 files
462 The Felis YAML files to validate.
464 Raises
465 ------
466 click.exceptions.Exit
467 Raised if any validation errors are found. The ``ValidationError``
468 which is thrown when a schema fails to validate will be logged as an
469 error message.
471 Notes
472 -----
473 All of the ``check`` flags are turned off by default and represent
474 optional validations controlled by the Pydantic context.
475 """
476 rc = 0
477 for file in files:
478 file_name = getattr(file, "name", None)
479 logger.info(f"Validating {file_name}")
480 try:
481 Schema.from_stream(
482 file,
483 context={
484 "check_description": check_description,
485 "check_redundant_datatypes": check_redundant_datatypes,
486 "check_tap_table_indexes": check_tap_table_indexes,
487 "check_tap_principal": check_tap_principal,
488 "id_generation": ctx.obj["id_generation"],
489 },
490 )
491 except ValidationError as e:
492 logger.error(e)
493 rc = 1
494 if rc:
495 raise click.exceptions.Exit(rc)
498@cli.command(
499 "diff",
500 help="""
501 Compare two schemas or a schema and a database for changes
503 Examples:
505 felis diff schema1.yaml schema2.yaml
507 felis diff -c alembic schema1.yaml schema2.yaml
509 felis diff --engine-url sqlite:///test.db schema.yaml
510 """,
511)
512@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
513@click.option(
514 "-c",
515 "--comparator",
516 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
517 help="Comparator to use for schema comparison",
518 default="deepdiff",
519)
520@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
521@click.argument("files", nargs=-1, type=click.File())
522@click.pass_context
523def diff(
524 ctx: click.Context,
525 engine_url: str | None,
526 comparator: str,
527 error_on_change: bool,
528 files: Iterable[IO[str]],
529) -> None:
530 schemas = [
531 Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
532 ]
534 diff: SchemaDiff
535 if len(schemas) == 2 and engine_url is None:
536 if comparator == "alembic":
537 db_context = create_database(schemas[0])
538 assert isinstance(db_context.engine, Engine)
539 diff = DatabaseDiff(schemas[1], db_context.engine)
540 else:
541 diff = FormattedSchemaDiff(schemas[0], schemas[1])
542 elif len(schemas) == 1 and engine_url is not None:
543 engine = create_engine(engine_url)
544 diff = DatabaseDiff(schemas[0], engine)
545 else:
546 raise click.ClickException(
547 "Invalid arguments - provide two schemas or a schema and a database engine URL"
548 )
550 diff.print()
552 if diff.has_changes and error_on_change:
553 raise click.ClickException("Schema was changed")
556@cli.command(
557 "dump",
558 help="""
559 Dump a schema file to YAML or JSON format
561 Example:
563 felis dump schema.yaml schema.json
565 felis dump schema.yaml schema_dump.yaml
566 """,
567)
568@click.option(
569 "--strip-ids/--no-strip-ids",
570 is_flag=True,
571 help="Strip IDs from the output schema",
572 default=False,
573)
574@click.argument("files", nargs=2, type=click.Path())
575@click.pass_context
576def dump(
577 ctx: click.Context,
578 strip_ids: bool,
579 files: list[str],
580) -> None:
581 if strip_ids:
582 logger.info("Stripping IDs from the output schema")
583 if files[1].endswith(".json"):
584 format = "json"
585 elif files[1].endswith(".yaml"):
586 format = "yaml"
587 else:
588 raise click.ClickException("Output file must have a .json or .yaml extension")
589 schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
590 with open(files[1], "w") as f:
591 if format == "yaml":
592 schema.dump_yaml(f, strip_ids=strip_ids)
593 elif format == "json":
594 schema.dump_json(f, strip_ids=strip_ids)
597if __name__ == "__main__": 597 ↛ 598line 597 didn't jump to line 598 because the condition on line 597 was never true
598 cli()