Coverage for python/felis/cli.py: 93%
226 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-29 01:46 -0700
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from pathlib import Path
29from typing import IO
31import click
32import yaml
33from pydantic import ValidationError
35from . import __version__
36from .browser import SiteConfig, render_static_site
37from .datamodel import Schema
38from .db.database_context import create_database_context
39from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
40from .metadata import create_metadata
41from .tap_schema import DataLoader, MetadataInserter, TableManager
43__all__ = ["cli"]
45logger = logging.getLogger(__name__)
47loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
50@click.group()
51@click.version_option(__version__)
52@click.option(
53 "--log-level",
54 type=click.Choice(loglevel_choices),
55 envvar="FELIS_LOGLEVEL",
56 help="Felis log level",
57 default=logging.getLevelName(logging.INFO),
58)
59@click.option(
60 "--log-file",
61 type=click.Path(),
62 envvar="FELIS_LOGFILE",
63 help="Felis log file path",
64)
65@click.option(
66 "--id-generation/--no-id-generation",
67 is_flag=True,
68 help="Generate IDs for all objects that do not have them",
69 default=True,
70)
71@click.option(
72 "--column-ref-index-increment",
73 type=int,
74 help="Automatically set 'tap:column_index' on column references, using the specified increment "
75 "(must be at least 1)",
76 default=None,
77)
78@click.pass_context
79def cli(
80 ctx: click.Context,
81 log_level: str,
82 log_file: str | None,
83 id_generation: bool,
84 column_ref_index_increment: int | None,
85) -> None:
86 """Felis command line tools"""
87 ctx.ensure_object(dict)
89 # Configure logging on the felis package logger directly.
90 # Do not use basicConfig() as it is a no-op when handlers already exist
91 # (e.g., when running under pytest).
92 felis_logger = logging.getLogger("felis")
93 felis_logger.setLevel(log_level)
94 if log_file:
95 handler: logging.Handler = logging.FileHandler(log_file)
96 else:
97 handler = logging.StreamHandler()
98 handler.setLevel(log_level)
99 handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
100 felis_logger.addHandler(handler)
102 # Configure ID generation (flag can only turn it off)
103 ctx.obj["id_generation"] = id_generation
104 if ctx.obj["id_generation"]:
105 logger.info("ID generation is enabled")
106 else:
107 logger.info("ID generation is disabled")
109 # Configure automatic indexing of column references (optional)
110 if column_ref_index_increment is not None and column_ref_index_increment < 1:
111 raise click.ClickException("column_ref_index_increment must be at least 1")
112 ctx.obj["column_ref_index_increment"] = column_ref_index_increment
113 if ctx.obj["column_ref_index_increment"] is not None:
114 logger.info(
115 "Automatic indexing of column references is enabled with increment %s",
116 ctx.obj["column_ref_index_increment"],
117 )
120@cli.command("create", help="Create database objects from the Felis file")
121@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
122@click.option("--schema-name", help="Alternate schema name to override Felis file")
123@click.option(
124 "--initialize",
125 is_flag=True,
126 help="Create the schema in the database if it does not exist (error if already exists)",
127)
128@click.option(
129 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
130)
131@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
132@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
133@click.option(
134 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
135)
136@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
137@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
138@click.argument("uri", type=str)
139@click.pass_context
140def create(
141 ctx: click.Context,
142 engine_url: str,
143 schema_name: str | None,
144 initialize: bool,
145 drop: bool,
146 echo: bool,
147 dry_run: bool,
148 output_file: IO[str] | None,
149 ignore_constraints: bool,
150 skip_indexes: bool,
151 uri: str,
152) -> None:
153 """Create database objects from the Felis file.
155 Parameters
156 ----------
157 engine_url
158 SQLAlchemy Engine URL.
159 schema_name
160 Alternate schema name to override Felis file.
161 initialize
162 Create the schema in the database if it does not exist.
163 drop
164 Drop schema if it already exists in the database.
165 echo
166 Echo database commands as they are executed.
167 dry_run
168 Dry run only to print out commands instead of executing.
169 output_file
170 Write SQL commands to a file instead of executing.
171 ignore_constraints
172 Ignore constraints when creating tables.
173 skip_indexes
174 Skip creating indexes when building metadata.
175 uri
176 URI of Felis file to read.
177 """
178 try:
179 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
181 metadata = create_metadata(
182 schema,
183 schema_name=schema_name,
184 ignore_constraints=ignore_constraints,
185 skip_indexes=skip_indexes,
186 engine_url=engine_url,
187 )
189 with create_database_context(
190 engine_url,
191 metadata,
192 echo=echo,
193 dry_run=dry_run,
194 output_file=output_file,
195 ) as db_ctx:
196 if drop and initialize:
197 raise ValueError("Cannot drop and initialize schema at the same time")
199 if drop: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true
200 logger.debug("Dropping schema if it exists")
201 db_ctx.drop()
202 initialize = True # If schema is dropped, it needs to be recreated.
204 if initialize: 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true
205 logger.debug("Creating schema if not exists")
206 db_ctx.initialize()
208 db_ctx.create_all()
210 except Exception as e:
211 logger.exception(e)
212 raise click.ClickException(str(e))
215@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
216@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
217@click.option("--schema-name", help="Alternate schema name to override Felis file")
218@click.argument("uri", type=str)
219@click.pass_context
220def create_indexes(
221 ctx: click.Context,
222 engine_url: str,
223 schema_name: str | None,
224 uri: str,
225) -> None:
226 """Create indexes from a Felis YAML file in a target database.
228 Parameters
229 ----------
230 engine_url
231 SQLAlchemy Engine URL.
232 file
233 Felis file to read.
234 """
235 try:
236 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
238 metadata = create_metadata(schema, schema_name=schema_name, engine_url=engine_url)
240 with create_database_context(engine_url, metadata) as db_ctx:
241 db_ctx.create_indexes()
242 except Exception as e:
243 logger.exception(e)
244 raise click.ClickException("Error creating indexes: " + str(e))
247@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
248@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
249@click.option("--schema-name", help="Alternate schema name to override Felis file")
250@click.argument("uri", type=str)
251@click.pass_context
252def drop_indexes(
253 ctx: click.Context,
254 engine_url: str,
255 schema_name: str | None,
256 uri: str,
257) -> None:
258 """Drop indexes from a Felis YAML file in a target database.
260 Parameters
261 ----------
262 engine_url
263 SQLAlchemy Engine URL.
264 schema-name
265 Alternate schema name to override Felis file.
266 file
267 Felis file to read.
268 """
269 try:
270 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
272 metadata = create_metadata(schema, schema_name=schema_name, engine_url=engine_url)
273 with create_database_context(engine_url, metadata) as db:
274 db.drop_indexes()
275 except Exception as e:
276 logger.exception(e)
277 raise click.ClickException("Error dropping indexes: " + str(e))
280@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
281@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
282@click.option(
283 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
284)
285@click.option(
286 "--tap-tables-postfix",
287 "-p",
288 help="Postfix which is applied to standard TAP_SCHEMA table names",
289 default="",
290)
291@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
292@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
293@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
294@click.option(
295 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
296)
297@click.option(
298 "--force-unbounded-arraysize",
299 is_flag=True,
300 help="Use unbounded arraysize by default for all variable length string columns"
301 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
302) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
303@click.option(
304 "--unique-keys",
305 "-u",
306 is_flag=True,
307 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
308 default=False,
309)
310@click.argument("uri", type=str)
311@click.pass_context
312def load_tap_schema(
313 ctx: click.Context,
314 engine_url: str,
315 tap_schema_name: str,
316 tap_tables_postfix: str,
317 tap_schema_index: int,
318 dry_run: bool,
319 echo: bool,
320 output_file: IO[str] | None,
321 force_unbounded_arraysize: bool,
322 unique_keys: bool,
323 uri: str,
324) -> None:
325 """Load TAP metadata from a Felis file.
327 Parameters
328 ----------
329 engine_url
330 SQLAlchemy Engine URL.
331 tap_tables_postfix
332 Postfix which is applied to standard TAP_SCHEMA table names.
333 tap_schema_index
334 TAP_SCHEMA index of the schema in this environment.
335 dry_run
336 Execute dry run only. Does not insert any data.
337 echo
338 Print out the generated insert statements to stdout.
339 output_file
340 Output file for writing generated SQL.
341 file
342 Felis file to read.
344 Notes
345 -----
346 The TAP_SCHEMA database must already exist or the command will fail. This
347 command will not initialize the TAP_SCHEMA tables.
348 """
349 # Create TableManager with automatic dialect detection
350 mgr = TableManager(
351 engine_url=engine_url,
352 schema_name=tap_schema_name,
353 table_name_postfix=tap_tables_postfix,
354 )
356 # Create DatabaseContext using TableManager's metadata
357 with create_database_context(
358 engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
359 ) as db_ctx:
360 schema = Schema.from_uri(
361 uri,
362 context={
363 "id_generation": ctx.obj["id_generation"],
364 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
365 "force_unbounded_arraysize": force_unbounded_arraysize,
366 },
367 )
369 DataLoader(
370 schema,
371 mgr,
372 db_context=db_ctx,
373 tap_schema_index=tap_schema_index,
374 dry_run=dry_run,
375 print_sql=echo,
376 output_file=output_file,
377 unique_keys=unique_keys,
378 ).load()
381@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
382@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
383@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
384@click.option(
385 "--extensions",
386 type=str,
387 default=None,
388 help=(
389 "Optional path to extensions YAML file (system path or resource:// URI). "
390 "If not provided, no extensions will be applied. "
391 "Example (default packaged extensions): "
392 "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
393 ),
394)
395@click.option(
396 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
397)
398@click.option(
399 "--insert-metadata/--no-insert-metadata",
400 is_flag=True,
401 help="Insert metadata describing TAP_SCHEMA itself",
402 default=True,
403)
404@click.pass_context
405def init_tap_schema(
406 ctx: click.Context,
407 engine_url: str,
408 tap_schema_name: str,
409 extensions: str | None,
410 tap_tables_postfix: str,
411 insert_metadata: bool,
412) -> None:
413 """Initialize a standard TAP_SCHEMA database.
415 Parameters
416 ----------
417 engine_url
418 SQLAlchemy Engine URL.
419 tap_schema_name
420 Name of the TAP_SCHEMA schema in the database.
421 extensions
422 Extensions YAML file.
423 tap_tables_postfix
424 Postfix which is applied to standard TAP_SCHEMA table names.
425 insert_metadata
426 Insert metadata describing TAP_SCHEMA itself.
427 If set to False, only the TAP_SCHEMA tables will be created, but no
428 metadata will be inserted.
429 """
430 # Create TableManager with automatic dialect detection
431 mgr = TableManager(
432 engine_url=engine_url,
433 schema_name=tap_schema_name,
434 table_name_postfix=tap_tables_postfix,
435 extensions_path=extensions,
436 )
438 # Create DatabaseContext using TableManager's metadata
439 with create_database_context(engine_url, mgr.metadata) as db_ctx:
440 mgr.initialize_database(db_context=db_ctx)
441 if insert_metadata: 441 ↛ exitline 441 didn't jump to the function exit
442 MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
445@cli.command("validate", help="Validate one or more Felis YAML files")
446@click.option(
447 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
448)
449@click.option(
450 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
451)
452@click.option(
453 "--check-tap-table-indexes",
454 is_flag=True,
455 help="Check that every table has a unique TAP table index",
456 default=False,
457)
458@click.option(
459 "--check-tap-principal",
460 is_flag=True,
461 help="Check that at least one column per table is flagged as TAP principal",
462 default=False,
463)
464@click.argument("uris", nargs=-1, type=str)
465@click.pass_context
466def validate(
467 ctx: click.Context,
468 check_description: bool,
469 check_redundant_datatypes: bool,
470 check_tap_table_indexes: bool,
471 check_tap_principal: bool,
472 uris: Iterable[str],
473) -> None:
474 """Validate one or more felis YAML files.
476 Parameters
477 ----------
478 check_description
479 Check that all objects have a valid description.
480 check_redundant_datatypes
481 Check for redundant type overrides.
482 check_tap_table_indexes
483 Check that every table has a unique TAP table index.
484 check_tap_principal
485 Check that at least one column per table is flagged as TAP principal.
486 uris
487 A list of URIs representing the Felis YAML files to validate. These can
488 be relative or absolute file paths or a URI with a supported scheme
489 such as ``resource://`` for package resources.
491 Raises
492 ------
493 click.exceptions.Exit
494 Raised if any validation errors are found. The ``ValidationError``
495 which is thrown when a schema fails to validate will be logged as an
496 error message.
498 Notes
499 -----
500 All of the ``check`` flags are turned off by default and represent
501 optional validations controlled by the Pydantic context.
502 """
503 rc = 0
504 for uri in uris:
505 logger.info("Validating schema at '%s'", uri)
506 try:
507 Schema.from_uri(
508 uri,
509 context={
510 "check_description": check_description,
511 "check_redundant_datatypes": check_redundant_datatypes,
512 "check_tap_table_indexes": check_tap_table_indexes,
513 "check_tap_principal": check_tap_principal,
514 "id_generation": ctx.obj["id_generation"],
515 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
516 },
517 )
518 logger.info("Successfully validated schema at '%s'", uri)
519 except ValidationError as e:
520 logger.error(e)
521 rc = 1
522 if rc:
523 raise click.exceptions.Exit(rc)
526@cli.command(
527 "diff",
528 help="""
529 Compare two schemas or a schema and a database for changes
531 Examples:
533 felis diff schema1.yaml schema2.yaml
535 felis diff -c alembic schema1.yaml schema2.yaml
537 felis diff --engine-url sqlite:///test.db schema.yaml
538 """,
539)
540@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
541@click.option(
542 "-c",
543 "--comparator",
544 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
545 help="Comparator to use for schema comparison",
546 default="deepdiff",
547)
548@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
549@click.argument("uris", nargs=-1, type=str)
550@click.pass_context
551def diff(
552 ctx: click.Context,
553 engine_url: str | None,
554 comparator: str,
555 error_on_change: bool,
556 uris: Iterable[str],
557) -> None:
558 uri_list = list(uris)
559 schemas = [Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]}) for uri in uri_list]
560 diff: SchemaDiff
561 if len(schemas) == 2:
562 if comparator == "alembic":
563 metadata = create_metadata(schemas[0], engine_url=engine_url)
564 with create_database_context(
565 engine_url if engine_url else "sqlite:///:memory:", metadata
566 ) as db_ctx:
567 db_ctx.initialize()
568 db_ctx.create_all()
569 diff = DatabaseDiff(schemas[1], db_ctx.engine)
570 else:
571 diff = FormattedSchemaDiff(schemas[0], schemas[1])
572 elif len(schemas) == 1 and engine_url is not None:
573 # Create minimal metadata for the context manager
574 from sqlalchemy import MetaData
576 metadata = MetaData()
578 with create_database_context(engine_url, metadata) as db_ctx:
579 diff = DatabaseDiff(schemas[0], db_ctx.engine)
580 else:
581 raise click.ClickException(
582 "Invalid arguments - provide two schemas or a single schema and a database engine URL"
583 )
585 diff.print()
587 if diff.has_changes and error_on_change:
588 raise click.ClickException("Schema was changed")
591@cli.command(
592 "dump",
593 help="""
594 Dump a schema file to YAML or JSON format
596 Example:
598 felis dump schema.yaml schema.json
600 felis dump schema.yaml schema_dump.yaml
601 """,
602)
603@click.option(
604 "--strip-ids/--no-strip-ids",
605 is_flag=True,
606 help="Strip IDs from the output schema",
607 default=False,
608)
609@click.option(
610 "--dereference-resources/--no-dereference-resources",
611 is_flag=True,
612 help="Remove any column references from resources and inline the full column definitions in the output",
613 default=False,
614)
615@click.option(
616 "--sort-columns/--no-sort-columns",
617 is_flag=True,
618 help="Sort columns alphabetically by name in the output",
619 default=False,
620)
621@click.argument("uris", nargs=2, type=str)
622@click.pass_context
623def dump(
624 ctx: click.Context,
625 strip_ids: bool,
626 dereference_resources: bool,
627 sort_columns: bool,
628 uris: list[str],
629) -> None:
630 if strip_ids:
631 logger.info("Stripping IDs from the output schema")
632 if sort_columns:
633 logger.info("Sorting output columns alphabetically")
634 if uris[1].endswith(".json"):
635 format = "json"
636 elif uris[1].endswith(".yaml"):
637 format = "yaml"
638 else:
639 raise click.ClickException("Output file must have a .json or .yaml extension")
640 schema = Schema.from_uri(
641 uris[0],
642 context={
643 "id_generation": ctx.obj["id_generation"],
644 "dereference_resources": dereference_resources,
645 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
646 },
647 )
648 with open(uris[1], "w") as f:
649 if format == "yaml":
650 schema.dump_yaml(f, strip_ids=strip_ids, sort_columns=sort_columns)
651 elif format == "json": 651 ↛ 653line 651 didn't jump to line 653
652 schema.dump_json(f, strip_ids=strip_ids, sort_columns=sort_columns)
653 logger.info("Dumped '%s' to '%s'", uris[0], uris[1])
656@cli.command("browser", help="Generate a static schema browser site from one or more Felis schemas")
657@click.option(
658 "--output-dir",
659 type=click.Path(file_okay=False, dir_okay=True),
660 required=True,
661 help="Output directory for generated static HTML; files already in it are kept",
662)
663@click.option(
664 "--config",
665 "config_file",
666 type=click.Path(exists=True, dir_okay=False),
667 help="Site configuration file (YAML) setting how the site looks",
668)
669@click.argument("uris", nargs=-1, type=str, required=True)
670@click.pass_context
671def browser(
672 ctx: click.Context,
673 output_dir: str,
674 config_file: str | None,
675 uris: tuple[str, ...],
676) -> None:
677 """Generate a static HTML browser for one or more Felis schemas.
679 Parameters
680 ----------
681 output_dir
682 The directory to write the site to.
683 config_file
684 The site configuration file, if any.
685 uris
686 URIs of the Felis schemas to include. These can be relative or
687 absolute file paths or a URI with a supported scheme such as
688 ``resource://`` for package resources.
689 """
690 try:
691 config = SiteConfig.from_file(Path(config_file)) if config_file else SiteConfig()
692 except (ValidationError, yaml.YAMLError, OSError) as e:
693 raise click.ClickException(f"Invalid site configuration '{config_file}': {e}") from e
695 schemas: list[Schema] = []
696 # Sorted, so which of two schemas with the same name gets the "-2" in its
697 # page names doesn't depend on the order the files are given in.
698 for uri in sorted(uris):
699 try:
700 schema = Schema.from_uri(
701 uri,
702 context={
703 "id_generation": ctx.obj["id_generation"],
704 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
705 },
706 )
707 except ValidationError as e:
708 raise click.ClickException(f"Schema validation failed for '{uri}': {e}") from e
709 except Exception as e:
710 raise click.ClickException(f"Failed loading schema '{uri}': {e}") from e
711 schemas.append(schema)
713 try:
714 render_static_site(
715 schemas=schemas,
716 output_dir=Path(output_dir),
717 config=config,
718 )
719 except Exception as e:
720 raise click.ClickException(f"Failed rendering browser site: {e}") from e
722 logger.info("Generated a browser site with %d schema(s) at '%s'", len(schemas), output_dir)
725if __name__ == "__main__": 725 ↛ 726line 725 didn't jump to line 726 because the condition on line 725 was never true
726 cli()