Coverage for python / felis / cli.py: 40%
189 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-30 08:42 +0000
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
33from . import __version__
34from .datamodel import Schema
35from .db.database_context import create_database_context
36from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
37from .metadata import create_metadata
38from .tap_schema import DataLoader, MetadataInserter, TableManager
40__all__ = ["cli"]
42logger = logging.getLogger("felis")
44loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
47@click.group()
48@click.version_option(__version__)
49@click.option(
50 "--log-level",
51 type=click.Choice(loglevel_choices),
52 envvar="FELIS_LOGLEVEL",
53 help="Felis log level",
54 default=logging.getLevelName(logging.INFO),
55)
56@click.option(
57 "--log-file",
58 type=click.Path(),
59 envvar="FELIS_LOGFILE",
60 help="Felis log file path",
61)
62@click.option(
63 "--id-generation/--no-id-generation",
64 is_flag=True,
65 help="Generate IDs for all objects that do not have them",
66 default=True,
67)
68@click.option(
69 "--column-ref-index-increment",
70 type=int,
71 help="Automatically set 'tap:column_index' on column references, using the specified increment "
72 "(must be at least 1)",
73 default=None,
74)
75@click.pass_context
76def cli(
77 ctx: click.Context,
78 log_level: str,
79 log_file: str | None,
80 id_generation: bool,
81 column_ref_index_increment: int | None,
82) -> None:
83 """Felis command line tools"""
84 ctx.ensure_object(dict)
86 # Configure logging (must come first)
87 if log_file:
88 logging.basicConfig(filename=log_file, level=log_level)
89 else:
90 logging.basicConfig(level=log_level)
92 # Configure ID generation (flag can only turn it off)
93 ctx.obj["id_generation"] = id_generation
94 if ctx.obj["id_generation"]:
95 logger.info("ID generation is enabled")
96 else:
97 logger.info("ID generation is disabled")
99 # Configure automatic indexing of column references (optional)
100 if column_ref_index_increment is not None and column_ref_index_increment < 1:
101 raise click.ClickException("column_ref_index_increment must be at least 1")
102 ctx.obj["column_ref_index_increment"] = column_ref_index_increment
103 if ctx.obj["column_ref_index_increment"] is not None:
104 logger.info(
105 f"Automatic indexing of column references is enabled with increment "
106 f"{ctx.obj['column_ref_index_increment']}"
107 )
110@cli.command("create", help="Create database objects from the Felis file")
111@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
112@click.option("--schema-name", help="Alternate schema name to override Felis file")
113@click.option(
114 "--initialize",
115 is_flag=True,
116 help="Create the schema in the database if it does not exist (error if already exists)",
117)
118@click.option(
119 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
120)
121@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
122@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
123@click.option(
124 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
125)
126@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
127@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
128@click.argument("file", type=click.File())
129@click.pass_context
130def create(
131 ctx: click.Context,
132 engine_url: str,
133 schema_name: str | None,
134 initialize: bool,
135 drop: bool,
136 echo: bool,
137 dry_run: bool,
138 output_file: IO[str] | None,
139 ignore_constraints: bool,
140 skip_indexes: bool,
141 file: IO[str],
142) -> None:
143 """Create database objects from the Felis file.
145 Parameters
146 ----------
147 engine_url
148 SQLAlchemy Engine URL.
149 schema_name
150 Alternate schema name to override Felis file.
151 initialize
152 Create the schema in the database if it does not exist.
153 drop
154 Drop schema if it already exists in the database.
155 echo
156 Echo database commands as they are executed.
157 dry_run
158 Dry run only to print out commands instead of executing.
159 output_file
160 Write SQL commands to a file instead of executing.
161 ignore_constraints
162 Ignore constraints when creating tables.
163 skip_indexes
164 Skip creating indexes when building metadata.
165 file
166 Felis file to read.
167 """
168 try:
169 metadata = create_metadata(
170 file,
171 id_generation=ctx.obj["id_generation"],
172 schema_name=schema_name,
173 ignore_constraints=ignore_constraints,
174 skip_indexes=skip_indexes,
175 engine_url=engine_url,
176 )
178 with create_database_context(
179 engine_url,
180 metadata,
181 echo=echo,
182 dry_run=dry_run,
183 output_file=output_file,
184 ) as db_ctx:
185 if drop and initialize:
186 raise ValueError("Cannot drop and initialize schema at the same time")
188 if drop:
189 logger.debug("Dropping schema if it exists")
190 db_ctx.drop()
191 initialize = True # If schema is dropped, it needs to be recreated.
193 if initialize:
194 logger.debug("Creating schema if not exists")
195 db_ctx.initialize()
197 db_ctx.create_all()
199 except Exception as e:
200 logger.exception(e)
201 raise click.ClickException(str(e))
204@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
205@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
206@click.option("--schema-name", help="Alternate schema name to override Felis file")
207@click.argument("file", type=click.File())
208@click.pass_context
209def create_indexes(
210 ctx: click.Context,
211 engine_url: str,
212 schema_name: str | None,
213 file: IO[str],
214) -> None:
215 """Create indexes from a Felis YAML file in a target database.
217 Parameters
218 ----------
219 engine_url
220 SQLAlchemy Engine URL.
221 file
222 Felis file to read.
223 """
224 try:
225 metadata = create_metadata(
226 file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
227 )
228 with create_database_context(engine_url, metadata) as db_ctx:
229 db_ctx.create_indexes()
230 except Exception as e:
231 logger.exception(e)
232 raise click.ClickException("Error creating indexes: " + str(e))
235@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
236@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
237@click.option("--schema-name", help="Alternate schema name to override Felis file")
238@click.argument("file", type=click.File())
239@click.pass_context
240def drop_indexes(
241 ctx: click.Context,
242 engine_url: str,
243 schema_name: str | None,
244 file: IO[str],
245) -> None:
246 """Drop indexes from a Felis YAML file in a target database.
248 Parameters
249 ----------
250 engine_url
251 SQLAlchemy Engine URL.
252 schema-name
253 Alternate schema name to override Felis file.
254 file
255 Felis file to read.
256 """
257 try:
258 metadata = create_metadata(
259 file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
260 )
261 with create_database_context(engine_url, metadata) as db:
262 db.drop_indexes()
263 except Exception as e:
264 logger.exception(e)
265 raise click.ClickException("Error dropping indexes: " + str(e))
268@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
269@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
270@click.option(
271 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
272)
273@click.option(
274 "--tap-tables-postfix",
275 "-p",
276 help="Postfix which is applied to standard TAP_SCHEMA table names",
277 default="",
278)
279@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
280@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
281@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
282@click.option(
283 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
284)
285@click.option(
286 "--force-unbounded-arraysize",
287 is_flag=True,
288 help="Use unbounded arraysize by default for all variable length string columns"
289 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
290) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
291@click.option(
292 "--unique-keys",
293 "-u",
294 is_flag=True,
295 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
296 default=False,
297)
298@click.argument("file", type=click.File())
299@click.pass_context
300def load_tap_schema(
301 ctx: click.Context,
302 engine_url: str,
303 tap_schema_name: str,
304 tap_tables_postfix: str,
305 tap_schema_index: int,
306 dry_run: bool,
307 echo: bool,
308 output_file: IO[str] | None,
309 force_unbounded_arraysize: bool,
310 unique_keys: bool,
311 file: IO[str],
312) -> None:
313 """Load TAP metadata from a Felis file.
315 Parameters
316 ----------
317 engine_url
318 SQLAlchemy Engine URL.
319 tap_tables_postfix
320 Postfix which is applied to standard TAP_SCHEMA table names.
321 tap_schema_index
322 TAP_SCHEMA index of the schema in this environment.
323 dry_run
324 Execute dry run only. Does not insert any data.
325 echo
326 Print out the generated insert statements to stdout.
327 output_file
328 Output file for writing generated SQL.
329 file
330 Felis file to read.
332 Notes
333 -----
334 The TAP_SCHEMA database must already exist or the command will fail. This
335 command will not initialize the TAP_SCHEMA tables.
336 """
337 # Create TableManager with automatic dialect detection
338 mgr = TableManager(
339 engine_url=engine_url,
340 schema_name=tap_schema_name,
341 table_name_postfix=tap_tables_postfix,
342 )
344 # Create DatabaseContext using TableManager's metadata
345 with create_database_context(
346 engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
347 ) as db_ctx:
348 schema = Schema.from_stream(
349 file,
350 context={
351 "id_generation": ctx.obj["id_generation"],
352 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
353 "force_unbounded_arraysize": force_unbounded_arraysize,
354 },
355 )
357 DataLoader(
358 schema,
359 mgr,
360 db_context=db_ctx,
361 tap_schema_index=tap_schema_index,
362 dry_run=dry_run,
363 print_sql=echo,
364 output_file=output_file,
365 unique_keys=unique_keys,
366 ).load()
369@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
370@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
371@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
372@click.option(
373 "--extensions",
374 type=str,
375 default=None,
376 help=(
377 "Optional path to extensions YAML file (system path or resource:// URI). "
378 "If not provided, no extensions will be applied. "
379 "Example (default packaged extensions): "
380 "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
381 ),
382)
383@click.option(
384 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
385)
386@click.option(
387 "--insert-metadata/--no-insert-metadata",
388 is_flag=True,
389 help="Insert metadata describing TAP_SCHEMA itself",
390 default=True,
391)
392@click.pass_context
393def init_tap_schema(
394 ctx: click.Context,
395 engine_url: str,
396 tap_schema_name: str,
397 extensions: str | None,
398 tap_tables_postfix: str,
399 insert_metadata: bool,
400) -> None:
401 """Initialize a standard TAP_SCHEMA database.
403 Parameters
404 ----------
405 engine_url
406 SQLAlchemy Engine URL.
407 tap_schema_name
408 Name of the TAP_SCHEMA schema in the database.
409 extensions
410 Extensions YAML file.
411 tap_tables_postfix
412 Postfix which is applied to standard TAP_SCHEMA table names.
413 insert_metadata
414 Insert metadata describing TAP_SCHEMA itself.
415 If set to False, only the TAP_SCHEMA tables will be created, but no
416 metadata will be inserted.
417 """
418 # Create TableManager with automatic dialect detection
419 mgr = TableManager(
420 engine_url=engine_url,
421 schema_name=tap_schema_name,
422 table_name_postfix=tap_tables_postfix,
423 extensions_path=extensions,
424 )
426 # Create DatabaseContext using TableManager's metadata
427 with create_database_context(engine_url, mgr.metadata) as db_ctx:
428 mgr.initialize_database(db_context=db_ctx)
429 if insert_metadata:
430 MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
433@cli.command("validate", help="Validate one or more Felis YAML files")
434@click.option(
435 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
436)
437@click.option(
438 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
439)
440@click.option(
441 "--check-tap-table-indexes",
442 is_flag=True,
443 help="Check that every table has a unique TAP table index",
444 default=False,
445)
446@click.option(
447 "--check-tap-principal",
448 is_flag=True,
449 help="Check that at least one column per table is flagged as TAP principal",
450 default=False,
451)
452@click.argument("files", nargs=-1, type=click.File())
453@click.pass_context
454def validate(
455 ctx: click.Context,
456 check_description: bool,
457 check_redundant_datatypes: bool,
458 check_tap_table_indexes: bool,
459 check_tap_principal: bool,
460 files: Iterable[IO[str]],
461) -> None:
462 """Validate one or more felis YAML files.
464 Parameters
465 ----------
466 check_description
467 Check that all objects have a valid description.
468 check_redundant_datatypes
469 Check for redundant type overrides.
470 check_tap_table_indexes
471 Check that every table has a unique TAP table index.
472 check_tap_principal
473 Check that at least one column per table is flagged as TAP principal.
474 files
475 The Felis YAML files to validate.
477 Raises
478 ------
479 click.exceptions.Exit
480 Raised if any validation errors are found. The ``ValidationError``
481 which is thrown when a schema fails to validate will be logged as an
482 error message.
484 Notes
485 -----
486 All of the ``check`` flags are turned off by default and represent
487 optional validations controlled by the Pydantic context.
488 """
489 rc = 0
490 for file in files:
491 file_name = getattr(file, "name", None)
492 logger.info(f"Validating {file_name}")
493 try:
494 Schema.from_stream(
495 file,
496 context={
497 "check_description": check_description,
498 "check_redundant_datatypes": check_redundant_datatypes,
499 "check_tap_table_indexes": check_tap_table_indexes,
500 "check_tap_principal": check_tap_principal,
501 "id_generation": ctx.obj["id_generation"],
502 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
503 },
504 )
505 logger.info(f"Successfully validated {file_name}")
506 except ValidationError as e:
507 logger.error(e)
508 rc = 1
509 if rc:
510 raise click.exceptions.Exit(rc)
513@cli.command(
514 "diff",
515 help="""
516 Compare two schemas or a schema and a database for changes
518 Examples:
520 felis diff schema1.yaml schema2.yaml
522 felis diff -c alembic schema1.yaml schema2.yaml
524 felis diff --engine-url sqlite:///test.db schema.yaml
525 """,
526)
527@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
528@click.option(
529 "-c",
530 "--comparator",
531 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
532 help="Comparator to use for schema comparison",
533 default="deepdiff",
534)
535@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
536@click.argument("files", nargs=-1, type=click.File())
537@click.pass_context
538def diff(
539 ctx: click.Context,
540 engine_url: str | None,
541 comparator: str,
542 error_on_change: bool,
543 files: Iterable[IO[str]],
544) -> None:
545 files_list = list(files)
546 schemas = [
547 Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files_list
548 ]
549 diff: SchemaDiff
550 if len(schemas) == 2:
551 if comparator == "alembic":
552 # Reset file stream to beginning before re-reading
553 files_list[0].seek(0)
554 metadata = create_metadata(
555 files_list[0], id_generation=ctx.obj["id_generation"], engine_url=engine_url
556 )
557 with create_database_context(
558 engine_url if engine_url else "sqlite:///:memory:", metadata
559 ) as db_ctx:
560 db_ctx.initialize()
561 db_ctx.create_all()
562 diff = DatabaseDiff(schemas[1], db_ctx.engine)
563 else:
564 diff = FormattedSchemaDiff(schemas[0], schemas[1])
565 elif len(schemas) == 1 and engine_url is not None:
566 # Create minimal metadata for the context manager
567 from sqlalchemy import MetaData
569 metadata = MetaData()
571 with create_database_context(engine_url, metadata) as db_ctx:
572 diff = DatabaseDiff(schemas[0], db_ctx.engine)
573 else:
574 raise click.ClickException(
575 "Invalid arguments - provide two schemas or a single schema and a database engine URL"
576 )
578 diff.print()
580 if diff.has_changes and error_on_change:
581 raise click.ClickException("Schema was changed")
584@cli.command(
585 "dump",
586 help="""
587 Dump a schema file to YAML or JSON format
589 Example:
591 felis dump schema.yaml schema.json
593 felis dump schema.yaml schema_dump.yaml
594 """,
595)
596@click.option(
597 "--strip-ids/--no-strip-ids",
598 is_flag=True,
599 help="Strip IDs from the output schema",
600 default=False,
601)
602@click.option(
603 "--dereference-resources/--no-dereference-resources",
604 is_flag=True,
605 help="Remove any column references from resources and inline the full column definitions in the output",
606 default=False,
607)
608@click.argument("files", nargs=2, type=click.Path())
609@click.pass_context
610def dump(
611 ctx: click.Context,
612 strip_ids: bool,
613 dereference_resources: bool,
614 files: list[str],
615) -> None:
616 if strip_ids:
617 logger.info("Stripping IDs from the output schema")
618 if files[1].endswith(".json"):
619 format = "json"
620 elif files[1].endswith(".yaml"):
621 format = "yaml"
622 else:
623 raise click.ClickException("Output file must have a .json or .yaml extension")
624 schema = Schema.from_uri(
625 files[0],
626 context={"id_generation": ctx.obj["id_generation"], "dereference_resources": dereference_resources},
627 )
628 with open(files[1], "w") as f:
629 if format == "yaml":
630 schema.dump_yaml(f, strip_ids=strip_ids)
631 elif format == "json":
632 schema.dump_json(f, strip_ids=strip_ids)
635if __name__ == "__main__": 635 ↛ 636line 635 didn't jump to line 636 because the condition on line 635 was never true
636 cli()