Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions datafaker/db_utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Utility functions."""

import gzip
import io
import re
from collections.abc import Callable, Mapping
from pathlib import Path
from types import ModuleType
Expand All @@ -22,6 +24,7 @@
)
from sqlalchemy.engine.interfaces import DBAPIConnection
from sqlalchemy.exc import (
ArgumentError,
IntegrityError,
NoReferencedTableError,
NoSuchModuleError,
Expand Down Expand Up @@ -60,6 +63,11 @@
MAKE_VOCAB_PROGRESS_REPORT_EVERY = 10000


def _hide_password(dsn: str) -> str:
"""Replace the password in a database address with ***."""
return re.sub(r"(://[^:/@]*:)[^@]*@", r"\1***@", dsn)


def table_row_count(table: Table, conn: Connection) -> int:
"""
Count the rows in the table.
Expand Down Expand Up @@ -162,11 +170,17 @@ def create_db_engine(
logger.error("Failed to connect to the database: %s", exc)
logger.error("Perhaps the dialect '%s' is invalid.", db_dsn.split(":")[0])
raise Exit(1) from exc
except ArgumentError as exc:
logger.error(
"The database address is not valid. Expected something like"
" postgresql://user:password@host:5432/database"
)
raise Exit(1) from exc
except ModuleNotFoundError as exc:
logger.error("Failed to connect to the database: %s", exc)
raise Exit(1) from exc
except ValueError as exc:
logger.error("DSN %s is malformed: %s", db_dsn, exc)
logger.error("DSN %s is malformed: %s", _hide_password(db_dsn), exc)
raise Exit(1) from exc

settings = {}
Expand Down Expand Up @@ -561,7 +575,7 @@ def sorted_non_vocabulary_tables(metadata: MetaData, config: Mapping) -> list[Ta
table_names = set(metadata.tables.keys()).difference(
get_vocabulary_table_names(config) | get_ignored_table_names(config)
)
(sorted_tables, cycles) = topological_sort(
sorted_tables, cycles = topological_sort(
table_names, lambda tn: get_related_table_names(metadata.tables[tn])
)
for cycle in cycles:
Expand Down
71 changes: 39 additions & 32 deletions datafaker/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Entrypoint for the datafaker package."""

import asyncio
import importlib.metadata
import io
Expand Down Expand Up @@ -46,6 +47,7 @@
conf_logger,
generators_require_stats,
get_flag,
load_yaml_file,
logger,
read_config_file,
)
Expand Down Expand Up @@ -103,28 +105,36 @@ def load_metadata_config(
:return: A dict representing the ``orm.yaml`` file, with the tables
the ``config`` says to ignore removed.
"""
with orm_file_path.open(encoding="utf-8") as orm_fh:
meta_dict = yaml.load(orm_fh, yaml.Loader)
if not isinstance(meta_dict, dict):
return {}
tables_dict = meta_dict.get("tables", {})
if config is not None and "tables" in config:
tables_config = config["tables"]
# Remove ignored tables
for name, table_config in tables_config.items():
if get_flag(table_config, "ignore"):
tables_dict.pop(name, None)
# Remove foreign keys to ignored tables
for table_meta in tables_dict.values():
for column_meta in table_meta.get("columns", {}).values():
if isinstance(column_meta, dict) and "foreign_keys" in column_meta:
filtered_fks = [
fk
for fk in column_meta["foreign_keys"]
if not should_ignore_fk(tables_config, fk)
]
column_meta["foreign_keys"] = filtered_fks
return meta_dict
meta_dict = load_yaml_file(
orm_file_path,
"ORM file",
"Please run make-tables first.",
yaml.SafeLoader,
)
if not isinstance(meta_dict, dict):
logger.error(
"ORM file '%s' is empty or invalid. Please run make-tables --force to recreate it.",
orm_file_path,
)
raise Exit(1)
tables_dict = meta_dict.get("tables", {})
if config is not None and "tables" in config:
tables_config = config["tables"]
# Remove ignored tables
for name, table_config in tables_config.items():
if get_flag(table_config, "ignore"):
tables_dict.pop(name, None)
# Remove foreign keys to ignored tables
for table_meta in tables_dict.values():
for column_meta in table_meta.get("columns", {}).values():
if isinstance(column_meta, dict) and "foreign_keys" in column_meta:
filtered_fks = [
fk
for fk in column_meta["foreign_keys"]
if not should_ignore_fk(tables_config, fk)
]
column_meta["foreign_keys"] = filtered_fks
return meta_dict


def load_metadata(orm_file_path: Path, config: dict | None = None) -> MetaData:
Expand Down Expand Up @@ -254,9 +264,8 @@ def configure_tables(
logger.debug("Configuring tables in %s.", config_file)
config = {}
if config_file.exists():
config = yaml.load(
config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader
)
config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader)

# we don't pass config here so that no tables are ignored
meta_dict = load_metadata_config(orm_file)
metadata = dict_to_metadata(meta_dict, None)
Expand Down Expand Up @@ -299,9 +308,7 @@ def configure_generators(
logger.debug("Configuring generators in %s.", config_file)
config = {}
if config_file.exists():
config = yaml.load(
config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader
)
config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader)
meta_dict = load_metadata_config(orm_file)
metadata = dict_to_metadata(meta_dict, None)
config_updated = update_config_generators(
Expand Down Expand Up @@ -339,9 +346,7 @@ def configure_missing(
logger.debug("Configuring missingness in %s.", config_file)
config: dict[str, Any] = {}
if config_file.exists():
config_any = yaml.load(
config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader
)
config_any = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader)
if isinstance(config_any, dict):
config = config_any
meta_dict = load_metadata_config(orm_file, config)
Expand Down Expand Up @@ -520,6 +525,8 @@ def create_data(
"row" if row_count == 1 else "rows",
)
return
except Exit:
raise
except RuntimeError as e:
logger.error(e.args[0])
except SettingsError as e:
Expand Down Expand Up @@ -665,7 +672,7 @@ def validate_config(
"""Validate the format of a config file."""
logger.debug("Validating config file: %s.", config_file)

config = yaml.load(config_file.read_text(encoding="UTF-8"), Loader=yaml.SafeLoader)
config = load_yaml_file(config_file, "Config file", "", yaml.SafeLoader)
schema_config = json.loads(CONFIG_SCHEMA_PATH.read_text(encoding="UTF-8"))
try:
validate(config, schema_config)
Expand Down
45 changes: 42 additions & 3 deletions datafaker/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Utility functions."""

import ast
import importlib.util
import io
Expand All @@ -19,6 +20,7 @@
from jsonschema.exceptions import ValidationError
from jsonschema.validators import validate
from sqlalchemy.engine import make_url
from typer import Exit

from datafaker.settings import SettingsError

Expand All @@ -43,6 +45,38 @@ def iterable(cls) -> Iterable[T]:
return (x for x in e)


def load_yaml_file(
path: Path,
description: str,
fix_hint: str,
loader: Any = yaml.SafeLoader,
) -> Any:
"""Load a YAML file, exiting with a helpful message if it cannot be read.

Args:
path: The file to load.
description: What the file is, for error messages, e.g. "Config file".
fix_hint: What the user should do if the file is missing.
loader: The PyYAML loader to use.

Returns:
The parsed contents of the file.
"""
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError as exc:
logger.error("%s '%s' not found. %s", description, path, fix_hint)
raise Exit(1) from exc
except (PermissionError, IsADirectoryError) as exc:
logger.error("Could not read %s '%s': %s", description, path, exc)
raise Exit(1) from exc
try:
return yaml.load(text, loader)
except yaml.YAMLError as exc:
logger.error("%s '%s' is not valid YAML: %s", description, path, exc)
raise Exit(1) from exc


def read_config_file(path: Path) -> dict:
"""Read a config file, warning if it is invalid.

Expand All @@ -52,8 +86,12 @@ def read_config_file(path: Path) -> dict:
Returns:
The config file as a dictionary.
"""
with path.open("r", encoding="utf8") as f:
config = yaml.safe_load(f)
config = load_yaml_file(
path,
"Config file",
"Please run configure-tables first.",
yaml.SafeLoader,
)

if not isinstance(config, dict):
logger.error(
Expand All @@ -66,6 +104,7 @@ def read_config_file(path: Path) -> dict:
validate(config, schema_config)
except ValidationError as e:
logger.error("The config file is invalid: %s", e.message)
raise Exit(1) from e

return config

Expand Down Expand Up @@ -357,7 +396,7 @@ def get_ignored_table_names(config: Mapping) -> set[str]:


def get_columns_assigned(
row_generator_config: Mapping[str, Any]
row_generator_config: Mapping[str, Any],
) -> Generator[str, None, None]:
"""
Get the columns assigned in a ``row_generators[n]`` stanza.
Expand Down
9 changes: 6 additions & 3 deletions tests/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Tests for the utils module."""

import os
import sys
import tempfile
Expand All @@ -8,6 +9,7 @@

from sqlalchemy import Column, Integer, insert
from sqlalchemy.orm import declarative_base
from typer import Exit

from datafaker.db_utils import download_table
from datafaker.utils import (
Expand Down Expand Up @@ -109,10 +111,11 @@ def test_download_table(self) -> None:
class TestReadConfig(DatafakerTestCase):
"""Tests for the read_config_file function."""

def test_warns_of_invalid_config(self) -> None:
"""Test that we get a warning if the config is invalid."""
def test_exits_on_invalid_config(self) -> None:
"""Test that an invalid config logs an error and exits."""
with patch("datafaker.utils.logger") as mock_logger:
read_config_file(Path("tests/examples/invalid_config.yaml"))
with self.assertRaises(Exit):
read_config_file(Path("tests/examples/invalid_config.yaml"))
mock_logger.error.assert_called_with(
"The config file is invalid: %s", "'a' is not of type 'integer'"
)
Expand Down
Loading