Coverage for python / felis / cli.py: 41%
181 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-14 23:37 +0000
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
33from . import __version__
34from .datamodel import Schema
35from .db.database_context import create_database_context
36from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
37from .metadata import create_metadata
38from .tap_schema import DataLoader, MetadataInserter, TableManager
40__all__ = ["cli"]
42logger = logging.getLogger("felis")
44loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
47@click.group()
48@click.version_option(__version__)
49@click.option(
50 "--log-level",
51 type=click.Choice(loglevel_choices),
52 envvar="FELIS_LOGLEVEL",
53 help="Felis log level",
54 default=logging.getLevelName(logging.INFO),
55)
56@click.option(
57 "--log-file",
58 type=click.Path(),
59 envvar="FELIS_LOGFILE",
60 help="Felis log file path",
61)
62@click.option(
63 "--id-generation/--no-id-generation",
64 is_flag=True,
65 help="Generate IDs for all objects that do not have them",
66 default=True,
67)
68@click.pass_context
69def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
70 """Felis command line tools"""
71 ctx.ensure_object(dict)
72 ctx.obj["id_generation"] = id_generation
73 if ctx.obj["id_generation"]:
74 logger.info("ID generation is enabled")
75 else:
76 logger.info("ID generation is disabled")
77 if log_file:
78 logging.basicConfig(filename=log_file, level=log_level)
79 else:
80 logging.basicConfig(level=log_level)
83@cli.command("create", help="Create database objects from the Felis file")
84@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
85@click.option("--schema-name", help="Alternate schema name to override Felis file")
86@click.option(
87 "--initialize",
88 is_flag=True,
89 help="Create the schema in the database if it does not exist (error if already exists)",
90)
91@click.option(
92 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
93)
94@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
95@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
96@click.option(
97 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
98)
99@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
100@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
101@click.argument("file", type=click.File())
102@click.pass_context
103def create(
104 ctx: click.Context,
105 engine_url: str,
106 schema_name: str | None,
107 initialize: bool,
108 drop: bool,
109 echo: bool,
110 dry_run: bool,
111 output_file: IO[str] | None,
112 ignore_constraints: bool,
113 skip_indexes: bool,
114 file: IO[str],
115) -> None:
116 """Create database objects from the Felis file.
118 Parameters
119 ----------
120 engine_url
121 SQLAlchemy Engine URL.
122 schema_name
123 Alternate schema name to override Felis file.
124 initialize
125 Create the schema in the database if it does not exist.
126 drop
127 Drop schema if it already exists in the database.
128 echo
129 Echo database commands as they are executed.
130 dry_run
131 Dry run only to print out commands instead of executing.
132 output_file
133 Write SQL commands to a file instead of executing.
134 ignore_constraints
135 Ignore constraints when creating tables.
136 skip_indexes
137 Skip creating indexes when building metadata.
138 file
139 Felis file to read.
140 """
141 try:
142 metadata = create_metadata(
143 file,
144 id_generation=ctx.obj["id_generation"],
145 schema_name=schema_name,
146 ignore_constraints=ignore_constraints,
147 skip_indexes=skip_indexes,
148 engine_url=engine_url,
149 )
151 with create_database_context(
152 engine_url,
153 metadata,
154 echo=echo,
155 dry_run=dry_run,
156 output_file=output_file,
157 ) as db_ctx:
158 if drop and initialize:
159 raise ValueError("Cannot drop and initialize schema at the same time")
161 if drop:
162 logger.debug("Dropping schema if it exists")
163 db_ctx.drop()
164 initialize = True # If schema is dropped, it needs to be recreated.
166 if initialize:
167 logger.debug("Creating schema if not exists")
168 db_ctx.initialize()
170 db_ctx.create_all()
172 except Exception as e:
173 logger.exception(e)
174 raise click.ClickException(str(e))
177@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
178@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
179@click.option("--schema-name", help="Alternate schema name to override Felis file")
180@click.argument("file", type=click.File())
181@click.pass_context
182def create_indexes(
183 ctx: click.Context,
184 engine_url: str,
185 schema_name: str | None,
186 file: IO[str],
187) -> None:
188 """Create indexes from a Felis YAML file in a target database.
190 Parameters
191 ----------
192 engine_url
193 SQLAlchemy Engine URL.
194 file
195 Felis file to read.
196 """
197 try:
198 metadata = create_metadata(
199 file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
200 )
201 with create_database_context(engine_url, metadata) as db_ctx:
202 db_ctx.create_indexes()
203 except Exception as e:
204 logger.exception(e)
205 raise click.ClickException("Error creating indexes: " + str(e))
208@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
209@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
210@click.option("--schema-name", help="Alternate schema name to override Felis file")
211@click.argument("file", type=click.File())
212@click.pass_context
213def drop_indexes(
214 ctx: click.Context,
215 engine_url: str,
216 schema_name: str | None,
217 file: IO[str],
218) -> None:
219 """Drop indexes from a Felis YAML file in a target database.
221 Parameters
222 ----------
223 engine_url
224 SQLAlchemy Engine URL.
225 schema-name
226 Alternate schema name to override Felis file.
227 file
228 Felis file to read.
229 """
230 try:
231 metadata = create_metadata(
232 file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
233 )
234 with create_database_context(engine_url, metadata) as db:
235 db.drop_indexes()
236 except Exception as e:
237 logger.exception(e)
238 raise click.ClickException("Error dropping indexes: " + str(e))
241@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
242@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
243@click.option(
244 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
245)
246@click.option(
247 "--tap-tables-postfix",
248 "-p",
249 help="Postfix which is applied to standard TAP_SCHEMA table names",
250 default="",
251)
252@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
253@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
254@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
255@click.option(
256 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
257)
258@click.option(
259 "--force-unbounded-arraysize",
260 is_flag=True,
261 help="Use unbounded arraysize by default for all variable length string columns"
262 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
263) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
264@click.option(
265 "--unique-keys",
266 "-u",
267 is_flag=True,
268 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
269 default=False,
270)
271@click.argument("file", type=click.File())
272@click.pass_context
273def load_tap_schema(
274 ctx: click.Context,
275 engine_url: str,
276 tap_schema_name: str,
277 tap_tables_postfix: str,
278 tap_schema_index: int,
279 dry_run: bool,
280 echo: bool,
281 output_file: IO[str] | None,
282 force_unbounded_arraysize: bool,
283 unique_keys: bool,
284 file: IO[str],
285) -> None:
286 """Load TAP metadata from a Felis file.
288 Parameters
289 ----------
290 engine_url
291 SQLAlchemy Engine URL.
292 tap_tables_postfix
293 Postfix which is applied to standard TAP_SCHEMA table names.
294 tap_schema_index
295 TAP_SCHEMA index of the schema in this environment.
296 dry_run
297 Execute dry run only. Does not insert any data.
298 echo
299 Print out the generated insert statements to stdout.
300 output_file
301 Output file for writing generated SQL.
302 file
303 Felis file to read.
305 Notes
306 -----
307 The TAP_SCHEMA database must already exist or the command will fail. This
308 command will not initialize the TAP_SCHEMA tables.
309 """
310 # Create TableManager with automatic dialect detection
311 mgr = TableManager(
312 engine_url=engine_url,
313 schema_name=tap_schema_name,
314 table_name_postfix=tap_tables_postfix,
315 )
317 # Create DatabaseContext using TableManager's metadata
318 with create_database_context(
319 engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
320 ) as db_ctx:
321 schema = Schema.from_stream(
322 file,
323 context={
324 "id_generation": ctx.obj["id_generation"],
325 "force_unbounded_arraysize": force_unbounded_arraysize,
326 },
327 )
329 DataLoader(
330 schema,
331 mgr,
332 db_context=db_ctx,
333 tap_schema_index=tap_schema_index,
334 dry_run=dry_run,
335 print_sql=echo,
336 output_file=output_file,
337 unique_keys=unique_keys,
338 ).load()
341@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
342@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
343@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
344@click.option(
345 "--extensions",
346 type=str,
347 default=None,
348 help=(
349 "Optional path to extensions YAML file (system path or resource:// URI). "
350 "If not provided, no extensions will be applied. "
351 "Example (default packaged extensions): "
352 "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
353 ),
354)
355@click.option(
356 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
357)
358@click.option(
359 "--insert-metadata/--no-insert-metadata",
360 is_flag=True,
361 help="Insert metadata describing TAP_SCHEMA itself",
362 default=True,
363)
364@click.pass_context
365def init_tap_schema(
366 ctx: click.Context,
367 engine_url: str,
368 tap_schema_name: str,
369 extensions: str | None,
370 tap_tables_postfix: str,
371 insert_metadata: bool,
372) -> None:
373 """Initialize a standard TAP_SCHEMA database.
375 Parameters
376 ----------
377 engine_url
378 SQLAlchemy Engine URL.
379 tap_schema_name
380 Name of the TAP_SCHEMA schema in the database.
381 extensions
382 Extensions YAML file.
383 tap_tables_postfix
384 Postfix which is applied to standard TAP_SCHEMA table names.
385 insert_metadata
386 Insert metadata describing TAP_SCHEMA itself.
387 If set to False, only the TAP_SCHEMA tables will be created, but no
388 metadata will be inserted.
389 """
390 # Create TableManager with automatic dialect detection
391 mgr = TableManager(
392 engine_url=engine_url,
393 schema_name=tap_schema_name,
394 table_name_postfix=tap_tables_postfix,
395 extensions_path=extensions,
396 )
398 # Create DatabaseContext using TableManager's metadata
399 with create_database_context(engine_url, mgr.metadata) as db_ctx:
400 mgr.initialize_database(db_context=db_ctx)
401 if insert_metadata:
402 MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
405@cli.command("validate", help="Validate one or more Felis YAML files")
406@click.option(
407 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
408)
409@click.option(
410 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
411)
412@click.option(
413 "--check-tap-table-indexes",
414 is_flag=True,
415 help="Check that every table has a unique TAP table index",
416 default=False,
417)
418@click.option(
419 "--check-tap-principal",
420 is_flag=True,
421 help="Check that at least one column per table is flagged as TAP principal",
422 default=False,
423)
424@click.argument("files", nargs=-1, type=click.File())
425@click.pass_context
426def validate(
427 ctx: click.Context,
428 check_description: bool,
429 check_redundant_datatypes: bool,
430 check_tap_table_indexes: bool,
431 check_tap_principal: bool,
432 files: Iterable[IO[str]],
433) -> None:
434 """Validate one or more felis YAML files.
436 Parameters
437 ----------
438 check_description
439 Check that all objects have a valid description.
440 check_redundant_datatypes
441 Check for redundant type overrides.
442 check_tap_table_indexes
443 Check that every table has a unique TAP table index.
444 check_tap_principal
445 Check that at least one column per table is flagged as TAP principal.
446 files
447 The Felis YAML files to validate.
449 Raises
450 ------
451 click.exceptions.Exit
452 Raised if any validation errors are found. The ``ValidationError``
453 which is thrown when a schema fails to validate will be logged as an
454 error message.
456 Notes
457 -----
458 All of the ``check`` flags are turned off by default and represent
459 optional validations controlled by the Pydantic context.
460 """
461 rc = 0
462 for file in files:
463 file_name = getattr(file, "name", None)
464 logger.info(f"Validating {file_name}")
465 try:
466 Schema.from_stream(
467 file,
468 context={
469 "check_description": check_description,
470 "check_redundant_datatypes": check_redundant_datatypes,
471 "check_tap_table_indexes": check_tap_table_indexes,
472 "check_tap_principal": check_tap_principal,
473 "id_generation": ctx.obj["id_generation"],
474 },
475 )
476 except ValidationError as e:
477 logger.error(e)
478 rc = 1
479 if rc:
480 raise click.exceptions.Exit(rc)
483@cli.command(
484 "diff",
485 help="""
486 Compare two schemas or a schema and a database for changes
488 Examples:
490 felis diff schema1.yaml schema2.yaml
492 felis diff -c alembic schema1.yaml schema2.yaml
494 felis diff --engine-url sqlite:///test.db schema.yaml
495 """,
496)
497@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
498@click.option(
499 "-c",
500 "--comparator",
501 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
502 help="Comparator to use for schema comparison",
503 default="deepdiff",
504)
505@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
506@click.argument("files", nargs=-1, type=click.File())
507@click.pass_context
508def diff(
509 ctx: click.Context,
510 engine_url: str | None,
511 comparator: str,
512 error_on_change: bool,
513 files: Iterable[IO[str]],
514) -> None:
515 files_list = list(files)
516 schemas = [
517 Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files_list
518 ]
519 diff: SchemaDiff
520 if len(schemas) == 2:
521 if comparator == "alembic":
522 # Reset file stream to beginning before re-reading
523 files_list[0].seek(0)
524 metadata = create_metadata(
525 files_list[0], id_generation=ctx.obj["id_generation"], engine_url=engine_url
526 )
527 with create_database_context(
528 engine_url if engine_url else "sqlite:///:memory:", metadata
529 ) as db_ctx:
530 db_ctx.initialize()
531 db_ctx.create_all()
532 diff = DatabaseDiff(schemas[1], db_ctx.engine)
533 else:
534 diff = FormattedSchemaDiff(schemas[0], schemas[1])
535 elif len(schemas) == 1 and engine_url is not None:
536 # Create minimal metadata for the context manager
537 from sqlalchemy import MetaData
539 metadata = MetaData()
541 with create_database_context(engine_url, metadata) as db_ctx:
542 diff = DatabaseDiff(schemas[0], db_ctx.engine)
543 else:
544 raise click.ClickException(
545 "Invalid arguments - provide two schemas or a single schema and a database engine URL"
546 )
548 diff.print()
550 if diff.has_changes and error_on_change:
551 raise click.ClickException("Schema was changed")
554@cli.command(
555 "dump",
556 help="""
557 Dump a schema file to YAML or JSON format
559 Example:
561 felis dump schema.yaml schema.json
563 felis dump schema.yaml schema_dump.yaml
564 """,
565)
566@click.option(
567 "--strip-ids/--no-strip-ids",
568 is_flag=True,
569 help="Strip IDs from the output schema",
570 default=False,
571)
572@click.argument("files", nargs=2, type=click.Path())
573@click.pass_context
574def dump(
575 ctx: click.Context,
576 strip_ids: bool,
577 files: list[str],
578) -> None:
579 if strip_ids:
580 logger.info("Stripping IDs from the output schema")
581 if files[1].endswith(".json"):
582 format = "json"
583 elif files[1].endswith(".yaml"):
584 format = "yaml"
585 else:
586 raise click.ClickException("Output file must have a .json or .yaml extension")
587 schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
588 with open(files[1], "w") as f:
589 if format == "yaml":
590 schema.dump_yaml(f, strip_ids=strip_ids)
591 elif format == "json":
592 schema.dump_json(f, strip_ids=strip_ids)
595if __name__ == "__main__": 595 ↛ 596line 595 didn't jump to line 596 because the condition on line 595 was never true
596 cli()