diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index cf48d4b..d36ca00 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -1,6 +1,8 @@ """Utility functions.""" + import gzip import io +import re from collections.abc import Callable, Mapping from pathlib import Path from types import ModuleType @@ -22,6 +24,7 @@ ) from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import ( + ArgumentError, IntegrityError, NoReferencedTableError, NoSuchModuleError, @@ -60,6 +63,11 @@ MAKE_VOCAB_PROGRESS_REPORT_EVERY = 10000 +def _hide_password(dsn: str) -> str: + """Replace the password in a database address with ***.""" + return re.sub(r"(://[^:/@]*:)[^@]*@", r"\1***@", dsn) + + def table_row_count(table: Table, conn: Connection) -> int: """ Count the rows in the table. @@ -162,11 +170,17 @@ def create_db_engine( logger.error("Failed to connect to the database: %s", exc) logger.error("Perhaps the dialect '%s' is invalid.", db_dsn.split(":")[0]) raise Exit(1) from exc + except ArgumentError as exc: + logger.error( + "The database address is not valid. Expected something like" + " postgresql://user:password@host:5432/database" + ) + raise Exit(1) from exc except ModuleNotFoundError as exc: logger.error("Failed to connect to the database: %s", exc) raise Exit(1) from exc except ValueError as exc: - logger.error("DSN %s is malformed: %s", db_dsn, exc) + logger.error("DSN %s is malformed: %s", _hide_password(db_dsn), exc) raise Exit(1) from exc settings = {} @@ -561,7 +575,7 @@ def sorted_non_vocabulary_tables(metadata: MetaData, config: Mapping) -> list[Ta table_names = set(metadata.tables.keys()).difference( get_vocabulary_table_names(config) | get_ignored_table_names(config) ) - (sorted_tables, cycles) = topological_sort( + sorted_tables, cycles = topological_sort( table_names, lambda tn: get_related_table_names(metadata.tables[tn]) ) for cycle in cycles: diff --git a/datafaker/main.py b/datafaker/main.py index b180b97..f95cdfe 100644 --- a/datafaker/main.py +++ b/datafaker/main.py @@ -1,4 +1,5 @@ """Entrypoint for the datafaker package.""" + import asyncio import importlib.metadata import io @@ -46,6 +47,7 @@ conf_logger, generators_require_stats, get_flag, + load_yaml_file, logger, read_config_file, ) @@ -103,28 +105,36 @@ def load_metadata_config( :return: A dict representing the ``orm.yaml`` file, with the tables the ``config`` says to ignore removed. """ - with orm_file_path.open(encoding="utf-8") as orm_fh: - meta_dict = yaml.load(orm_fh, yaml.Loader) - if not isinstance(meta_dict, dict): - return {} - tables_dict = meta_dict.get("tables", {}) - if config is not None and "tables" in config: - tables_config = config["tables"] - # Remove ignored tables - for name, table_config in tables_config.items(): - if get_flag(table_config, "ignore"): - tables_dict.pop(name, None) - # Remove foreign keys to ignored tables - for table_meta in tables_dict.values(): - for column_meta in table_meta.get("columns", {}).values(): - if isinstance(column_meta, dict) and "foreign_keys" in column_meta: - filtered_fks = [ - fk - for fk in column_meta["foreign_keys"] - if not should_ignore_fk(tables_config, fk) - ] - column_meta["foreign_keys"] = filtered_fks - return meta_dict + meta_dict = load_yaml_file( + orm_file_path, + "ORM file", + "Please run make-tables first.", + yaml.SafeLoader, + ) + if not isinstance(meta_dict, dict): + logger.error( + "ORM file '%s' is empty or invalid. Please run make-tables --force to recreate it.", + orm_file_path, + ) + raise Exit(1) + tables_dict = meta_dict.get("tables", {}) + if config is not None and "tables" in config: + tables_config = config["tables"] + # Remove ignored tables + for name, table_config in tables_config.items(): + if get_flag(table_config, "ignore"): + tables_dict.pop(name, None) + # Remove foreign keys to ignored tables + for table_meta in tables_dict.values(): + for column_meta in table_meta.get("columns", {}).values(): + if isinstance(column_meta, dict) and "foreign_keys" in column_meta: + filtered_fks = [ + fk + for fk in column_meta["foreign_keys"] + if not should_ignore_fk(tables_config, fk) + ] + column_meta["foreign_keys"] = filtered_fks + return meta_dict def load_metadata(orm_file_path: Path, config: dict | None = None) -> MetaData: @@ -254,9 +264,8 @@ def configure_tables( logger.debug("Configuring tables in %s.", config_file) config = {} if config_file.exists(): - config = yaml.load( - config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader - ) + config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader) + # we don't pass config here so that no tables are ignored meta_dict = load_metadata_config(orm_file) metadata = dict_to_metadata(meta_dict, None) @@ -299,9 +308,7 @@ def configure_generators( logger.debug("Configuring generators in %s.", config_file) config = {} if config_file.exists(): - config = yaml.load( - config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader - ) + config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader) meta_dict = load_metadata_config(orm_file) metadata = dict_to_metadata(meta_dict, None) config_updated = update_config_generators( @@ -339,9 +346,7 @@ def configure_missing( logger.debug("Configuring missingness in %s.", config_file) config: dict[str, Any] = {} if config_file.exists(): - config_any = yaml.load( - config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader - ) + config_any = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader) if isinstance(config_any, dict): config = config_any meta_dict = load_metadata_config(orm_file, config) @@ -520,6 +525,8 @@ def create_data( "row" if row_count == 1 else "rows", ) return + except Exit: + raise except RuntimeError as e: logger.error(e.args[0]) except SettingsError as e: @@ -665,7 +672,7 @@ def validate_config( """Validate the format of a config file.""" logger.debug("Validating config file: %s.", config_file) - config = yaml.load(config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader) + config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader) schema_config = json.loads(CONFIG_SCHEMA_PATH.read_text(encoding="UTF-8")) try: validate(config, schema_config) diff --git a/datafaker/utils.py b/datafaker/utils.py index a6a1532..d0456e9 100644 --- a/datafaker/utils.py +++ b/datafaker/utils.py @@ -1,4 +1,5 @@ """Utility functions.""" + import ast import importlib.util import io @@ -19,6 +20,7 @@ from jsonschema.exceptions import ValidationError from jsonschema.validators import validate from sqlalchemy.engine import make_url +from typer import Exit from datafaker.settings import SettingsError @@ -43,6 +45,38 @@ def iterable(cls) -> Iterable[T]: return (x for x in e) +def load_yaml_file( + path: Path, + description: str, + fix_hint: str, + loader: Any = yaml.SafeLoader, +) -> Any: + """Load a YAML file, exiting with a helpful message if it cannot be read. + + Args: + path: The file to load. + description: What the file is, for error messages, e.g. "Config file". + fix_hint: What the user should do if the file is missing. + loader: The PyYAML loader to use. + + Returns: + The parsed contents of the file. + """ + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + logger.error("%s '%s' not found. %s", description, path, fix_hint) + raise Exit(1) from exc + except (PermissionError, IsADirectoryError) as exc: + logger.error("Could not read %s '%s': %s", description, path, exc) + raise Exit(1) from exc + try: + return yaml.load(text, loader) + except yaml.YAMLError as exc: + logger.error("%s '%s' is not valid YAML: %s", description, path, exc) + raise Exit(1) from exc + + def read_config_file(path: Path) -> dict: """Read a config file, warning if it is invalid. @@ -52,8 +86,12 @@ def read_config_file(path: Path) -> dict: Returns: The config file as a dictionary. """ - with path.open("r", encoding="utf8") as f: - config = yaml.safe_load(f) + config = load_yaml_file( + path, + "Config file", + "Please run configure-tables first.", + yaml.SafeLoader, + ) if not isinstance(config, dict): logger.error( @@ -66,6 +104,7 @@ def read_config_file(path: Path) -> dict: validate(config, schema_config) except ValidationError as e: logger.error("The config file is invalid: %s", e.message) + raise Exit(1) from e return config @@ -357,7 +396,7 @@ def get_ignored_table_names(config: Mapping) -> set[str]: def get_columns_assigned( - row_generator_config: Mapping[str, Any] + row_generator_config: Mapping[str, Any], ) -> Generator[str, None, None]: """ Get the columns assigned in a ``row_generators[n]`` stanza. diff --git a/tests/test_utils.py b/tests/test_utils.py index 2dded71..f1fff25 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ """Tests for the utils module.""" + import os import sys import tempfile @@ -8,6 +9,7 @@ from sqlalchemy import Column, Integer, insert from sqlalchemy.orm import declarative_base +from typer import Exit from datafaker.db_utils import download_table from datafaker.utils import ( @@ -109,10 +111,11 @@ def test_download_table(self) -> None: class TestReadConfig(DatafakerTestCase): """Tests for the read_config_file function.""" - def test_warns_of_invalid_config(self) -> None: - """Test that we get a warning if the config is invalid.""" + def test_exits_on_invalid_config(self) -> None: + """Test that an invalid config logs an error and exits.""" with patch("datafaker.utils.logger") as mock_logger: - read_config_file(Path("tests/examples/invalid_config.yaml")) + with self.assertRaises(Exit): + read_config_file(Path("tests/examples/invalid_config.yaml")) mock_logger.error.assert_called_with( "The config file is invalid: %s", "'a' is not of type 'integer'" )