From 50d5e2e3f2080f049058b30e801691d36b3cacd9 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:25:30 -0400 Subject: [PATCH 01/16] feat: add splits payload normalizer for splitter schemas --- DashAI/back/splitters/splits_payload.py | 86 +++++++++++++++++++++ tests/back/splitters/__init__.py | 0 tests/back/splitters/test_splits_payload.py | 73 +++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 DashAI/back/splitters/splits_payload.py create mode 100644 tests/back/splitters/__init__.py create mode 100644 tests/back/splitters/test_splits_payload.py diff --git a/DashAI/back/splitters/splits_payload.py b/DashAI/back/splitters/splits_payload.py new file mode 100644 index 000000000..f07b6e448 --- /dev/null +++ b/DashAI/back/splitters/splits_payload.py @@ -0,0 +1,86 @@ +"""Helpers to read the ``splits`` payload stored in a model session. + +The payload is written by the frontend as a flat object: the selected +splitter's schema parameters at the top level, plus the meta keys +``splitter_name``, ``splitType`` and ``splitted_indexes``. Sessions created +before the splitter forms were generated from the component schemas used a +different set of keys, so every reader normalizes the payload first. +""" + +from typing import Any, Dict, Type + +INDEX_KEY_BY_SPLIT = { + "train": "train_indexes", + "test": "test_indexes", + "validation": "val_indexes", +} + +META_KEYS = ("splitter_name", "splitType", "splitted_indexes", "seed") + + +def normalize_splits_payload(splits_data: Dict[str, Any]) -> Dict[str, Any]: + """Translate a legacy splits payload into the current contract. + + Two legacy shapes are handled: + + - ``seed`` instead of the schema's ``random_state``. + - ``train`` / ``test`` / ``validation`` holding index lists instead of + proportions, which is how manual and predefined holdout splits used to + be stored. + + Parameters + ---------- + splits_data : dict + The payload as stored in ``ModelSession.splits``. + + Returns + ------- + dict + A new dictionary following the current contract. The input is left + untouched. + """ + normalized = dict(splits_data) + + if "random_state" not in normalized and "seed" in normalized: + normalized["random_state"] = normalized["seed"] + + index_lists = { + split: normalized[split] + for split in INDEX_KEY_BY_SPLIT + if isinstance(normalized.get(split), list) + } + if index_lists: + splitted_indexes = dict(normalized.get("splitted_indexes") or {}) + for split, indexes in index_lists.items(): + splitted_indexes.setdefault(INDEX_KEY_BY_SPLIT[split], indexes) + del normalized[split] + normalized["splitted_indexes"] = splitted_indexes + normalized.setdefault("splitType", "manual") + + return normalized + + +def schema_placeholder_defaults(splitter_cls: Type) -> Dict[str, Any]: + """Collect the placeholder value declared for each schema parameter. + + The frontend seeds its form with these values, so filling them in makes a + partial payload (for example a manual split, which carries no + proportions) validate exactly as the form would have submitted it. + + Parameters + ---------- + splitter_cls : type + A splitter class exposing ``get_schema``. + + Returns + ------- + dict + Parameter name to placeholder value, skipping parameters that declare + no placeholder. + """ + properties = splitter_cls.get_schema().get("properties", {}) + return { + name: prop["placeholder"] + for name, prop in properties.items() + if "placeholder" in prop + } diff --git a/tests/back/splitters/__init__.py b/tests/back/splitters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/splitters/test_splits_payload.py b/tests/back/splitters/test_splits_payload.py new file mode 100644 index 000000000..dde5a071d --- /dev/null +++ b/tests/back/splitters/test_splits_payload.py @@ -0,0 +1,73 @@ +from DashAI.back.splitters.holdout import HoldoutSplitter +from DashAI.back.splitters.splits_payload import ( + normalize_splits_payload, + schema_placeholder_defaults, +) + + +def test_legacy_seed_becomes_random_state(): + payload = {"splitter_name": "HoldoutSplitter", "seed": 7, "train": 0.6} + assert normalize_splits_payload(payload)["random_state"] == 7 + + +def test_explicit_random_state_wins_over_seed(): + payload = {"seed": 7, "random_state": 13} + assert normalize_splits_payload(payload)["random_state"] == 13 + + +def test_legacy_index_lists_move_to_splitted_indexes(): + payload = { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "train": [0, 1, 2], + "test": [3], + "validation": [4], + } + result = normalize_splits_payload(payload) + assert result["splitted_indexes"] == { + "train_indexes": [0, 1, 2], + "test_indexes": [3], + "val_indexes": [4], + } + assert "train" not in result + assert "test" not in result + assert "validation" not in result + + +def test_legacy_index_lists_default_split_type_to_manual(): + payload = {"train": [0, 1], "test": [2], "validation": []} + assert normalize_splits_payload(payload)["splitType"] == "manual" + + +def test_new_payload_is_unchanged_and_idempotent(): + payload = { + "splitter_name": "HoldoutSplitter", + "splitType": "random", + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + once = normalize_splits_payload(payload) + assert once == payload + assert normalize_splits_payload(once) == payload + + +def test_normalize_does_not_mutate_input(): + payload = {"seed": 7} + normalize_splits_payload(payload) + assert payload == {"seed": 7} + + +def test_schema_placeholder_defaults_reads_placeholders(): + defaults = schema_placeholder_defaults(HoldoutSplitter) + assert defaults == { + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "random_state": 42, + } From 6fde19b0ee360fc6ef2b13f71df8b95e1ef06571 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:27:03 -0400 Subject: [PATCH 02/16] fix: validate holdout proportions and make them non-nullable --- DashAI/back/splitters/holdout.py | 33 ++++- tests/back/splitters/test_holdout_splitter.py | 119 ++++++++++++++++++ 2 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 tests/back/splitters/test_holdout_splitter.py diff --git a/DashAI/back/splitters/holdout.py b/DashAI/back/splitters/holdout.py index ba73efdd4..10ea84974 100644 --- a/DashAI/back/splitters/holdout.py +++ b/DashAI/back/splitters/holdout.py @@ -2,12 +2,13 @@ from typing import TYPE_CHECKING, Any, Dict, List, Tuple +from pydantic import model_validator + from DashAI.back.core.schema_fields import ( BaseSchema, bool_field, float_field, int_field, - none_type, schema_field, ) from DashAI.back.core.utils import MultilingualString @@ -23,7 +24,7 @@ class HoldoutSplitterSchema(BaseSchema): train: schema_field( - none_type(float_field(ge=0, le=1)), + float_field(ge=0, le=1), placeholder=0.6, description=MultilingualString( en="Proportion of the dataset assigned to the training partition.", @@ -37,7 +38,7 @@ class HoldoutSplitterSchema(BaseSchema): ), ) # type: ignore test: schema_field( - none_type(float_field(ge=0, le=1)), + float_field(ge=0, le=1), placeholder=0.2, description=MultilingualString( en="Proportion of the dataset assigned to the test partition.", @@ -51,7 +52,7 @@ class HoldoutSplitterSchema(BaseSchema): ), ) # type: ignore validation: schema_field( - none_type(float_field(ge=0, le=1)), + float_field(ge=0, le=1), placeholder=0.2, description=MultilingualString( en="Proportion of the dataset assigned to the validation partition.", @@ -128,6 +129,30 @@ class HoldoutSplitterSchema(BaseSchema): ), ) # type: ignore + @model_validator(mode="after") + def check_partitions(self): + """Validate that the three partitions describe the whole dataset. + + Returns + ------- + HoldoutSplitterSchema + The validated schema instance. + + Raises + ------ + ValueError + If the proportions do not sum to one, or if the training partition + is empty. + """ + total = self.train + self.test + self.validation + if abs(total - 1) > 1e-6: + raise ValueError( + f"train, test and validation proportions must sum to 1, got {total}." + ) + if self.train <= 0: + raise ValueError("The train proportion must be greater than 0.") + return self + class HoldoutSplitter(BaseSplitter): """Splitter that creates train, test, and validation partitions for holdout diff --git a/tests/back/splitters/test_holdout_splitter.py b/tests/back/splitters/test_holdout_splitter.py new file mode 100644 index 000000000..82392197e --- /dev/null +++ b/tests/back/splitters/test_holdout_splitter.py @@ -0,0 +1,119 @@ +import pandas as pd +import pytest +from pydantic import ValidationError + +from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset +from DashAI.back.splitters.holdout import HoldoutSplitter, HoldoutSplitterSchema +from DashAI.back.splitters.splits_payload import normalize_splits_payload + + +@pytest.fixture +def xy(): + dataset = to_dashai_dataset(pd.DataFrame({"a": list(range(10)), "y": [0, 1] * 5})) + return dataset.select_columns(["a"]), dataset.select_columns(["y"]) + + +def test_manual_payload_uses_given_indexes(xy): + x, y = xy + splitter = HoldoutSplitter( + { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "splitted_indexes": { + "train_indexes": [0, 1, 2, 3, 4, 5], + "test_indexes": [6, 7], + "val_indexes": [8, 9], + }, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + ) + x_split, _, indices = splitter.split(x, y) + + assert indices["train_indexes"] == [0, 1, 2, 3, 4, 5] + assert len(x_split["train"]) == 6 + assert len(x_split["test"]) == 2 + assert len(x_split["validation"]) == 2 + + +def test_legacy_manual_payload_splits_after_normalization(xy): + x, y = xy + legacy = { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "train": [0, 1, 2, 3, 4, 5], + "test": [6, 7], + "validation": [8, 9], + } + splitter = HoldoutSplitter(normalize_splits_payload(legacy)) + x_split, _, indices = splitter.split(x, y) + + assert indices["test_indexes"] == [6, 7] + assert len(x_split["train"]) == 6 + + +def test_random_state_from_payload_changes_the_split(xy): + x, y = xy + base = { + "splitType": "random", + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + } + first = HoldoutSplitter({**base, "random_state": 1}).split(x, y)[2] + same = HoldoutSplitter({**base, "random_state": 1}).split(x, y)[2] + other = HoldoutSplitter({**base, "random_state": 99}).split(x, y)[2] + + assert first["train_indexes"] == same["train_indexes"] + assert first["train_indexes"] != other["train_indexes"] + + +def test_schema_rejects_proportions_that_do_not_sum_to_one(): + with pytest.raises(ValidationError): + HoldoutSplitterSchema.model_validate( + { + "train": 0.8, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + ) + + +def test_schema_rejects_empty_train_partition(): + with pytest.raises(ValidationError): + HoldoutSplitterSchema.model_validate( + { + "train": 0.0, + "test": 0.5, + "validation": 0.5, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + ) + + +def test_schema_accepts_the_placeholder_configuration(): + schema = HoldoutSplitterSchema.model_validate( + { + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + ) + assert schema.train == 0.6 + + +def test_proportions_are_plain_numbers_in_the_json_schema(): + train = HoldoutSplitter.get_schema()["properties"]["train"] + assert train["type"] == "number" + assert "anyOf" not in train From 7aa10b88672e56a62a5b258448e003bf8cb0bd4a Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:28:04 -0400 Subject: [PATCH 03/16] feat: bound fold count and repeat count in splitter schemas --- DashAI/back/splitters/group_k_fold.py | 12 +++---- DashAI/back/splitters/k_fold.py | 12 +++---- DashAI/back/splitters/repeated_k_fold.py | 24 ++++++------- .../splitters/repeated_stratified_k_fold.py | 24 ++++++------- .../back/splitters/stratified_group_k_fold.py | 12 +++---- DashAI/back/splitters/stratified_k_fold.py | 12 +++---- .../splitters/test_fold_splitter_schemas.py | 35 +++++++++++++++++++ 7 files changed, 83 insertions(+), 48 deletions(-) create mode 100644 tests/back/splitters/test_fold_splitter_schemas.py diff --git a/DashAI/back/splitters/group_k_fold.py b/DashAI/back/splitters/group_k_fold.py index 358c9a2c4..97405dff5 100644 --- a/DashAI/back/splitters/group_k_fold.py +++ b/DashAI/back/splitters/group_k_fold.py @@ -23,14 +23,14 @@ class GroupKFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", diff --git a/DashAI/back/splitters/k_fold.py b/DashAI/back/splitters/k_fold.py index ace6e1531..d44bc51fd 100644 --- a/DashAI/back/splitters/k_fold.py +++ b/DashAI/back/splitters/k_fold.py @@ -21,14 +21,14 @@ class KFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", diff --git a/DashAI/back/splitters/repeated_k_fold.py b/DashAI/back/splitters/repeated_k_fold.py index 4536bfa66..584d235ad 100644 --- a/DashAI/back/splitters/repeated_k_fold.py +++ b/DashAI/back/splitters/repeated_k_fold.py @@ -16,14 +16,14 @@ class RepeatedKFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", @@ -34,26 +34,26 @@ class RepeatedKFoldSplitterSchema(BaseSchema): ), ) # type: ignore n_repeats: schema_field( - int_field(ge=2), + int_field(ge=2, le=10), placeholder=2, description=MultilingualString( en=( "Number of times the K-Fold procedure is repeated. Must be an " - "integer greater than or equal to 2." + "integer between 2 and 10." ), es=( "Número de veces que se repite el procedimiento K-Fold. Debe " - "ser un entero mayor o igual a 2." + "ser un entero entre 2 y 10." ), pt=( "Número de vezes que o procedimento K-Fold é repetido. Deve " - "ser um inteiro maior ou igual a 2." + "ser um inteiro entre 2 e 10." ), de=( "Anzahl der Wiederholungen des K-Fold-Verfahrens. Muss eine " - "ganze Zahl größer oder gleich 2 sein." + "ganze Zahl zwischen 2 und 10 sein." ), - zh="K折过程重复的次数,必须为大于或等于2的整数。", + zh="K折过程重复的次数,必须为2到10之间的整数。", ), alias=MultilingualString( en="Number of repeats", diff --git a/DashAI/back/splitters/repeated_stratified_k_fold.py b/DashAI/back/splitters/repeated_stratified_k_fold.py index 19490dd71..6c544e6ff 100644 --- a/DashAI/back/splitters/repeated_stratified_k_fold.py +++ b/DashAI/back/splitters/repeated_stratified_k_fold.py @@ -16,14 +16,14 @@ class RepeatedStratifiedKFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", @@ -34,26 +34,26 @@ class RepeatedStratifiedKFoldSplitterSchema(BaseSchema): ), ) # type: ignore n_repeats: schema_field( - int_field(ge=2), + int_field(ge=2, le=10), placeholder=2, description=MultilingualString( en=( "Number of times the Stratified K-Fold procedure is repeated. " - "Must be an integer greater than or equal to 2." + "Must be an integer between 2 and 10." ), es=( "Número de veces que se repite el procedimiento K-Fold " - "estratificado. Debe ser un entero mayor o igual a 2." + "estratificado. Debe ser un entero entre 2 y 10." ), pt=( "Número de vezes que o procedimento K-Fold estratificado é " - "repetido. Deve ser um inteiro maior ou igual a 2." + "repetido. Deve ser um inteiro entre 2 e 10." ), de=( "Anzahl der Wiederholungen des stratifizierten K-Fold-" - "Verfahrens. Muss eine ganze Zahl größer oder gleich 2 sein." + "Verfahrens. Muss eine ganze Zahl zwischen 2 und 10 sein." ), - zh="分层K折过程重复的次数,必须为大于或等于2的整数。", + zh="分层K折过程重复的次数,必须为2到10之间的整数。", ), alias=MultilingualString( en="Number of repeats", diff --git a/DashAI/back/splitters/stratified_group_k_fold.py b/DashAI/back/splitters/stratified_group_k_fold.py index 7448f778b..e67a75a45 100644 --- a/DashAI/back/splitters/stratified_group_k_fold.py +++ b/DashAI/back/splitters/stratified_group_k_fold.py @@ -23,14 +23,14 @@ class StratifiedGroupKFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", diff --git a/DashAI/back/splitters/stratified_k_fold.py b/DashAI/back/splitters/stratified_k_fold.py index 8ca45a5fb..6a6232354 100644 --- a/DashAI/back/splitters/stratified_k_fold.py +++ b/DashAI/back/splitters/stratified_k_fold.py @@ -21,14 +21,14 @@ class StratifiedKFoldSplitterSchema(BaseSchema): n_splits: schema_field( - int_field(ge=2), + int_field(ge=2, le=20), placeholder=5, description=MultilingualString( - en="Number of folds. Must be an integer greater than or equal to 2.", - es="Número de particiones. Debe ser un entero mayor o igual a 2.", - pt="Número de partições. Deve ser um inteiro maior ou igual a 2.", - de="Anzahl der Folds. Muss eine ganze Zahl größer oder gleich 2 sein.", - zh="折数,必须为大于或等于2的整数。", + en="Number of folds. Must be an integer between 2 and 20.", + es="Número de particiones. Debe ser un entero entre 2 y 20.", + pt="Número de partições. Deve ser um inteiro entre 2 e 20.", + de="Anzahl der Folds. Muss eine ganze Zahl zwischen 2 und 20 sein.", + zh="折数,必须为2到20之间的整数。", ), alias=MultilingualString( en="Number of folds", diff --git a/tests/back/splitters/test_fold_splitter_schemas.py b/tests/back/splitters/test_fold_splitter_schemas.py new file mode 100644 index 000000000..d06ef1120 --- /dev/null +++ b/tests/back/splitters/test_fold_splitter_schemas.py @@ -0,0 +1,35 @@ +import pytest + +from DashAI.back.splitters.group_k_fold import GroupKFoldSplitter +from DashAI.back.splitters.k_fold import KFoldSplitter +from DashAI.back.splitters.repeated_k_fold import RepeatedKFoldSplitter +from DashAI.back.splitters.repeated_stratified_k_fold import ( + RepeatedStratifiedKFoldSplitter, +) +from DashAI.back.splitters.stratified_group_k_fold import StratifiedGroupKFoldSplitter +from DashAI.back.splitters.stratified_k_fold import StratifiedKFoldSplitter + +FOLD_SPLITTERS = [ + KFoldSplitter, + StratifiedKFoldSplitter, + GroupKFoldSplitter, + StratifiedGroupKFoldSplitter, + RepeatedKFoldSplitter, + RepeatedStratifiedKFoldSplitter, +] + + +@pytest.mark.parametrize("splitter_cls", FOLD_SPLITTERS) +def test_n_splits_is_bounded(splitter_cls): + n_splits = splitter_cls.get_schema()["properties"]["n_splits"] + assert n_splits["minimum"] == 2 + assert n_splits["maximum"] == 20 + + +@pytest.mark.parametrize( + "splitter_cls", [RepeatedKFoldSplitter, RepeatedStratifiedKFoldSplitter] +) +def test_n_repeats_is_bounded(splitter_cls): + n_repeats = splitter_cls.get_schema()["properties"]["n_repeats"] + assert n_repeats["minimum"] == 2 + assert n_repeats["maximum"] == 10 From 1cd993862d927c814c03fcb4d3bc76c83bd792a5 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:32:56 -0400 Subject: [PATCH 04/16] fix: normalize splits payload before building the splitter --- DashAI/back/job/model_job.py | 4 ++ tests/back/api/test_jobs.py | 74 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/DashAI/back/job/model_job.py b/DashAI/back/job/model_job.py index e6a4ceaf7..a5f16905b 100644 --- a/DashAI/back/job/model_job.py +++ b/DashAI/back/job/model_job.py @@ -12,6 +12,7 @@ from DashAI.back.models.model_factory import ModelFactory from DashAI.back.optimizers.base_optimizer import BaseOptimizer from DashAI.back.splitters.base_splitter import BaseSplitter +from DashAI.back.splitters.splits_payload import normalize_splits_payload from DashAI.back.tasks.base_task import BaseTask if TYPE_CHECKING: @@ -334,6 +335,9 @@ def _prepare_dataset_and_components( splits_data = json.loads(model_session.splits) if run.split_indexes: splits_data["splitted_indexes"] = json.loads(run.split_indexes) + # Sessions created before the splits payload followed the splitter + # schema use different keys for the seed and for manual indexes. + splits_data = normalize_splits_payload(splits_data) except Exception as e: log.exception(e) raise JobError( diff --git a/tests/back/api/test_jobs.py b/tests/back/api/test_jobs.py index 532e70fc8..9dba816d7 100644 --- a/tests/back/api/test_jobs.py +++ b/tests/back/api/test_jobs.py @@ -317,3 +317,77 @@ def test_job_with_wrong_run(client: TestClient): ) assert response.status_code == 500, response.text assert response.status_code == 500, response.text + + +def test_legacy_manual_splits_still_train(client: TestClient, model_session_id: int): + """A session stored before the splits payload followed the splitter schema. + + Manual and predefined holdout splits used to be stored with the partition + indexes under the ``train`` / ``test`` / ``validation`` keys, which the + splitter reads as proportions. + """ + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + original_splits = model_session.splits + model_session.splits = json.dumps( + { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "train": [0, 1, 2, 3, 4, 5], + "test": [6, 7], + "validation": [8, 9], + } + ) + db.commit() + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "DummyModel", + "name": "LegacyManualSplitRun", + "parameters": {}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Run over a legacy manual split payload", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 201, response.text + run_id = response.json()["id"] + + try: + response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert response.status_code == 201, response.text + job_id = response.json()["id"] + + response = client.get(f"/api/v1/job/status/{job_id}") + assert response.status_code == 200, response.text + assert response.json()["status"] == "finished" + + with session_factory() as db: + run = db.get(Run, run_id) + assert json.loads(run.split_indexes) == { + "train_indexes": [0, 1, 2, 3, 4, 5], + "test_indexes": [6, 7], + "val_indexes": [8, 9], + } + finally: + with session_factory() as db: + db.get(ModelSession, model_session_id).splits = original_splits + db.commit() + client.delete(f"/api/v1/run/{run_id}") From bb86b559537f08dde32390e9af57edfe63879328 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:33:57 -0400 Subject: [PATCH 05/16] fix: read manual split indexes and random state in session prep --- .../dataloaders/classes/dashai_dataset.py | 35 +++++++---- .../test_prepare_for_model_session_splits.py | 59 +++++++++++++++++++ 2 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 tests/back/splitters/test_prepare_for_model_session_splits.py diff --git a/DashAI/back/dataloaders/classes/dashai_dataset.py b/DashAI/back/dataloaders/classes/dashai_dataset.py index 5db8d41f0..fe5852d0c 100644 --- a/DashAI/back/dataloaders/classes/dashai_dataset.py +++ b/DashAI/back/dataloaders/classes/dashai_dataset.py @@ -1601,10 +1601,13 @@ def prepare_for_model_session( The unified dataset to split and preprocess. splits : dict Split configuration with at least a ``"splitType"`` key - (``"manual"``, ``"predefined"``, or ``"random"``). For manual/ - predefined splits the dict must also contain ``"train"``, ``"test"``, - and ``"validation"`` index lists; for random splits it must contain - float proportions summing to 1.0. + (``"manual"``, ``"predefined"``, or ``"random"``). Manual and + predefined splits carry their row indexes in ``"splitted_indexes"`` + (``"train_indexes"``, ``"test_indexes"``, ``"val_indexes"``), or, for + sessions created before the splits payload followed the splitter + schema, in ``"train"``, ``"test"`` and ``"validation"`` index lists. + Random splits carry float proportions summing to 1.0 and an optional + ``"random_state"`` (``"seed"`` in older sessions). output_columns : list of str Names of the columns to use as model targets. @@ -1617,16 +1620,24 @@ def prepare_for_model_session( splitType = splits.get("splitType") if splitType == "manual" or splitType == "predefined": - splits_index = splits + # The indexes travel under "splitted_indexes"; sessions created before + # the splits payload followed the splitter schema stored them under the + # partition names instead. + splitted_indexes = splits.get("splitted_indexes") or {} + if splitted_indexes: + train_indexes = splitted_indexes.get("train_indexes", []) + test_indexes = splitted_indexes.get("test_indexes", []) + val_indexes = splitted_indexes.get("val_indexes", []) + else: + train_indexes = splits["train"] + test_indexes = splits["test"] + val_indexes = splits["validation"] prepared_dataset = split_dataset( dataset, - train_indexes=splits_index["train"], - test_indexes=splits_index["test"], - val_indexes=splits_index["validation"], + train_indexes=train_indexes, + test_indexes=test_indexes, + val_indexes=val_indexes, ) - train_indexes = splits_index["train"] - test_indexes = splits_index["test"] - val_indexes = splits_index["validation"] else: n = len(dataset) labels = None @@ -1658,7 +1669,7 @@ def prepare_for_model_session( float(splits["test"]), float(splits["validation"]), shuffle=splits.get("shuffle", False), - seed=splits.get("seed"), + seed=splits.get("random_state", splits.get("seed")), stratify=splits.get("stratify", False), labels=labels, ) diff --git a/tests/back/splitters/test_prepare_for_model_session_splits.py b/tests/back/splitters/test_prepare_for_model_session_splits.py new file mode 100644 index 000000000..98d21560a --- /dev/null +++ b/tests/back/splitters/test_prepare_for_model_session_splits.py @@ -0,0 +1,59 @@ +import pandas as pd + +from DashAI.back.dataloaders.classes.dashai_dataset import ( + prepare_for_model_session, + to_dashai_dataset, +) + + +def dataset(): + return to_dashai_dataset(pd.DataFrame({"a": list(range(10)), "y": [0, 1] * 5})) + + +def test_manual_split_reads_splitted_indexes(): + splits = { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "splitted_indexes": { + "train_indexes": [0, 1, 2, 3, 4, 5], + "test_indexes": [6, 7], + "val_indexes": [8, 9], + }, + } + prepared, indexes = prepare_for_model_session(dataset(), splits, ["y"]) + + assert indexes["train_indexes"] == [0, 1, 2, 3, 4, 5] + assert len(prepared["test"]) == 2 + assert len(prepared["validation"]) == 2 + + +def test_legacy_manual_split_still_reads_index_lists(): + splits = { + "splitType": "manual", + "train": [0, 1, 2, 3, 4, 5], + "test": [6, 7], + "validation": [8, 9], + } + prepared, indexes = prepare_for_model_session(dataset(), splits, ["y"]) + + assert indexes["test_indexes"] == [6, 7] + assert len(prepared["train"]) == 6 + + +def test_random_split_accepts_random_state_as_the_seed(): + splits = { + "splitType": "random", + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "shuffle": True, + "random_state": 7, + } + _, first = prepare_for_model_session(dataset(), splits, ["y"]) + _, again = prepare_for_model_session(dataset(), splits, ["y"]) + _, other = prepare_for_model_session( + dataset(), {**splits, "random_state": 99}, ["y"] + ) + + assert first["train_indexes"] == again["train_indexes"] + assert first["train_indexes"] != other["train_indexes"] From 582b8974a8254cc47709e7ca644337bf6ade8106 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:41:11 -0400 Subject: [PATCH 06/16] feat: validate session splits against the splitter schema --- .../api/api_v1/endpoints/model_sessions.py | 66 +++++++++++ DashAI/back/splitters/splits_payload.py | 10 +- .../test_model_session_splits_validation.py | 105 ++++++++++++++++++ tests/back/splitters/test_splits_payload.py | 10 ++ 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 tests/back/api/test_model_session_splits_validation.py diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index 46d170aac..175da91e8 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, Response, status from fastapi.exceptions import HTTPException from kink import di, inject +from pydantic import ValidationError from sqlalchemy import exc, select from DashAI.back.api.api_v1.schemas.model_sessions_params import ( @@ -11,6 +12,11 @@ ModelSessionParams, ) from DashAI.back.dependencies.database.models import Dataset, ModelSession +from DashAI.back.splitters.splits_payload import ( + META_KEYS, + normalize_splits_payload, + schema_placeholder_defaults, +) if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker @@ -169,11 +175,65 @@ async def validate_columns( return validation_response +def _validate_splits(splits: str, component_registry: "ComponentRegistry") -> None: + """Validate a splits payload against the schema of its splitter. + + The payload carries the splitter's schema parameters at the top level. Any + parameter the caller left out is filled with the schema placeholder, which + is the value the frontend form would have submitted, so a manual split + (which carries row indexes instead of proportions) validates too. + + Parameters + ---------- + splits : str + The splits payload, serialized as JSON. + component_registry : ComponentRegistry + Registry used to resolve the splitter named in the payload. + + Raises + ------ + HTTPException + If the payload is not valid JSON, names a splitter that is not + registered, or carries parameters the splitter schema rejects. + """ + import json + + try: + splits_data = normalize_splits_payload(json.loads(splits)) + except (TypeError, ValueError) as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"splits is not a valid JSON object: {e}", + ) from e + + splitter_name = splits_data.get("splitter_name") + if splitter_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Splitter {splitter_name} does not exist in the registry.", + ) + + splitter_class = component_registry[splitter_name]["class"] + parameters = { + key: value for key, value in splits_data.items() if key not in META_KEYS + } + try: + splitter_class.SCHEMA.model_validate( + {**schema_placeholder_defaults(splitter_class), **parameters} + ) + except ValidationError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Invalid configuration for splitter {splitter_name}: {e}", + ) from e + + @router.post("/", status_code=status.HTTP_201_CREATED) @inject async def create_model_session( params: ModelSessionParams, session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), ): """Create a new model session. @@ -184,6 +244,8 @@ async def create_model_session( session_factory : Callable[..., ContextManager[Session]] A factory that creates a context manager that handles a SQLAlchemy session. The generated session can be used to access and query the database. + component_registry : ComponentRegistry + Registry used to resolve the splitter named in the splits payload. Returns ------- @@ -192,6 +254,8 @@ async def create_model_session( Raises ------ + HTTPException + If the splits payload is not valid for the selected splitter. HTTPException If the dataset with id dataset_id is not registered in the DB. """ @@ -200,6 +264,8 @@ async def create_model_session( import pyarrow as pa import pyarrow.ipc as ipc + _validate_splits(params.splits, component_registry) + with session_factory() as db: try: dataset = db.get(Dataset, params.dataset_id) diff --git a/DashAI/back/splitters/splits_payload.py b/DashAI/back/splitters/splits_payload.py index f07b6e448..a301a54a9 100644 --- a/DashAI/back/splitters/splits_payload.py +++ b/DashAI/back/splitters/splits_payload.py @@ -9,6 +9,10 @@ from typing import Any, Dict, Type +# Before fold splitters existed the payload named no splitter at all: every +# split was a holdout split. +DEFAULT_SPLITTER_NAME = "HoldoutSplitter" + INDEX_KEY_BY_SPLIT = { "train": "train_indexes", "test": "test_indexes", @@ -21,8 +25,9 @@ def normalize_splits_payload(splits_data: Dict[str, Any]) -> Dict[str, Any]: """Translate a legacy splits payload into the current contract. - Two legacy shapes are handled: + Three legacy shapes are handled: + - No ``splitter_name``, from before fold splitters existed. - ``seed`` instead of the schema's ``random_state``. - ``train`` / ``test`` / ``validation`` holding index lists instead of proportions, which is how manual and predefined holdout splits used to @@ -41,6 +46,9 @@ def normalize_splits_payload(splits_data: Dict[str, Any]) -> Dict[str, Any]: """ normalized = dict(splits_data) + if not isinstance(normalized.get("splitter_name"), str): + normalized["splitter_name"] = DEFAULT_SPLITTER_NAME + if "random_state" not in normalized and "seed" in normalized: normalized["random_state"] = normalized["seed"] diff --git a/tests/back/api/test_model_session_splits_validation.py b/tests/back/api/test_model_session_splits_validation.py new file mode 100644 index 000000000..3820443a9 --- /dev/null +++ b/tests/back/api/test_model_session_splits_validation.py @@ -0,0 +1,105 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset + + +def _session_body(splits: dict, dataset: Dataset, name: str) -> dict: + return { + "name": name, + "dataset_id": dataset.id, + "task_name": "TabularClassificationTask", + "input_columns": ["SepalLengthCm"], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps(splits), + } + + +def test_rejects_proportions_that_do_not_sum_to_one( + client: TestClient, dataset_1: Dataset +): + splits = { + "splitter_name": "HoldoutSplitter", + "splitType": "random", + "train": 0.8, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + response = client.post( + "/api/v1/model-session/", + json=_session_body(splits, dataset_1, "bad proportions"), + ) + assert response.status_code == 422, response.text + assert "sum to 1" in response.text + + +def test_rejects_a_single_fold(client: TestClient, dataset_1: Dataset): + splits = { + "splitter_name": "KFoldSplitter", + "splitType": "cv", + "n_splits": 1, + "shuffle": True, + "random_state": 42, + } + body = _session_body(splits, dataset_1, "single fold") + body["evaluation_strategy"] = "CrossValidationEvaluationStrategy" + + response = client.post("/api/v1/model-session/", json=body) + assert response.status_code == 422, response.text + + +def test_rejects_an_unknown_splitter(client: TestClient, dataset_1: Dataset): + splits = {"splitter_name": "NotASplitter", "splitType": "random"} + response = client.post( + "/api/v1/model-session/", + json=_session_body(splits, dataset_1, "unknown splitter"), + ) + assert response.status_code == 422, response.text + + +def test_accepts_a_manual_split_without_proportions( + client: TestClient, dataset_1: Dataset +): + splits = { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "splitted_indexes": { + "train_indexes": [0, 1, 2], + "test_indexes": [3], + "val_indexes": [4], + }, + "stratify": False, + "shuffle": True, + "random_state": 42, + } + response = client.post( + "/api/v1/model-session/", + json=_session_body(splits, dataset_1, "manual split session"), + ) + assert response.status_code == 201, response.text + + +def test_accepts_a_legacy_seed_payload(client: TestClient, dataset_1: Dataset): + splits = { + "splitter_name": "HoldoutSplitter", + "splitType": "random", + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + "seed": 7, + } + response = client.post( + "/api/v1/model-session/", + json=_session_body(splits, dataset_1, "legacy seed session"), + ) + assert response.status_code == 201, response.text diff --git a/tests/back/splitters/test_splits_payload.py b/tests/back/splitters/test_splits_payload.py index dde5a071d..c00493660 100644 --- a/tests/back/splitters/test_splits_payload.py +++ b/tests/back/splitters/test_splits_payload.py @@ -55,6 +55,16 @@ def test_new_payload_is_unchanged_and_idempotent(): assert normalize_splits_payload(once) == payload +def test_missing_splitter_name_defaults_to_holdout(): + payload = {"train": 0.6, "test": 0.2, "validation": 0.2, "seed": 42} + assert normalize_splits_payload(payload)["splitter_name"] == "HoldoutSplitter" + + +def test_null_splitter_name_defaults_to_holdout(): + payload = {"splitter_name": None, "train": 0.6} + assert normalize_splits_payload(payload)["splitter_name"] == "HoldoutSplitter" + + def test_normalize_does_not_mutate_input(): payload = {"seed": 7} normalize_splits_payload(payload) From 181f733ab3c069d4753c2576eba64cfbc7a43e2f Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:49:56 -0400 Subject: [PATCH 07/16] feat: let schema forms omit hand-rendered fields --- DashAI/front/src/components/shared/FormSchema.jsx | 3 +++ .../front/src/components/shared/FormSchemaRenderFields.jsx | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/DashAI/front/src/components/shared/FormSchema.jsx b/DashAI/front/src/components/shared/FormSchema.jsx index f2269a85a..678f09de2 100644 --- a/DashAI/front/src/components/shared/FormSchema.jsx +++ b/DashAI/front/src/components/shared/FormSchema.jsx @@ -34,6 +34,7 @@ function FormSchema({ onValuesChange, showBorder = true, hideButtons = false, + omitFields, }) { const { formik, modelSchema, loading, handleUpdateSchema } = useFormSchema({ model, @@ -61,6 +62,7 @@ function FormSchema({ onFormSubmit={onFormSubmit} setError={setError} errorsMessage={errorsMessage} + omitFields={omitFields} /> @@ -92,6 +94,7 @@ FormSchema.propTypes = { onValuesChange: PropTypes.func, showBorder: PropTypes.bool, hideButtons: PropTypes.bool, + omitFields: PropTypes.arrayOf(PropTypes.string), }; export default FormSchema; diff --git a/DashAI/front/src/components/shared/FormSchemaRenderFields.jsx b/DashAI/front/src/components/shared/FormSchemaRenderFields.jsx index 3834579fa..c96ef53e4 100644 --- a/DashAI/front/src/components/shared/FormSchemaRenderFields.jsx +++ b/DashAI/front/src/components/shared/FormSchemaRenderFields.jsx @@ -57,6 +57,7 @@ function FormSchemaRenderFields({ setError, errorsMessage, spacing = 2, + omitFields = [], }) { if (!modelSchema) return null; @@ -76,6 +77,10 @@ function FormSchemaRenderFields({ const fields = []; for (const key in modelSchema) { + // Fields the caller renders by hand: a schema field whose input needs + // context the schema cannot carry, such as a dataset's column names. + if (omitFields.includes(key)) continue; + const fieldSchema = modelSchema[key]; const objName = key; const value = formik?.values?.[objName]; @@ -178,6 +183,7 @@ function FormSchemaRenderFields({ handleChange, setError, errorsMessage, + omitFields, ]); return {renderFields()}; @@ -192,6 +198,7 @@ FormSchemaRenderFields.propTypes = { setError: PropTypes.func, errorsMessage: PropTypes.object, spacing: PropTypes.number, + omitFields: PropTypes.arrayOf(PropTypes.string), }; export default FormSchemaRenderFields; From 9173ea1081a84572f027ee83edb02f1d8f0ea862 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 09:50:27 -0400 Subject: [PATCH 08/16] feat: add splits payload builder for the model session wizard --- DashAI/front/src/utils/splitsPayload.js | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 DashAI/front/src/utils/splitsPayload.js diff --git a/DashAI/front/src/utils/splitsPayload.js b/DashAI/front/src/utils/splitsPayload.js new file mode 100644 index 000000000..2b1b847fe --- /dev/null +++ b/DashAI/front/src/utils/splitsPayload.js @@ -0,0 +1,54 @@ +export const SPLIT_TYPES = { + RANDOM: "random", + MANUAL: "manual", + PREDEFINED: "predefined", + CV: "cv", +}; + +// Proportions describe a random split only. Manual and predefined splits carry +// the row indexes instead, and the backend splitter picks its index branch when +// no proportion is present, so sending both would be ambiguous. +const PROPORTION_KEYS = ["train", "test", "validation"]; + +const INDEX_MODES = [SPLIT_TYPES.MANUAL, SPLIT_TYPES.PREDEFINED]; + +/** + * Build the splits payload stored on a model session. + * + * The selected splitter's schema parameters stay at the top level, next to the + * three meta keys the schemas do not own: splitter_name, splitType and + * splitted_indexes. Keeping the schema keys verbatim is what stops the payload + * and the splitter from drifting apart. + * + * @param {string} splitterName registry name of the selected splitter + * @param {string} splitType one of SPLIT_TYPES + * @param {object} params values submitted by the schema generated form + * @param {object} indexes row indexes per partition, for the index modes + * @returns {object} the payload to serialize into the model session splits + */ +export const buildSplitsPayload = ({ + splitterName, + splitType, + params = {}, + indexes = null, +}) => { + const isIndexMode = INDEX_MODES.includes(splitType); + + const payload = { splitter_name: splitterName, splitType }; + + Object.entries(params ?? {}).forEach(([key, value]) => { + if (value === undefined) return; + if (isIndexMode && PROPORTION_KEYS.includes(key)) return; + payload[key] = value; + }); + + if (isIndexMode) { + payload.splitted_indexes = { + train_indexes: indexes?.train ?? [], + test_indexes: indexes?.test ?? [], + val_indexes: indexes?.validation ?? [], + }; + } + + return payload; +}; From bb9e7a2f9b98be431a573c3c505ac2763a863779 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 10:07:07 -0400 Subject: [PATCH 09/16] feat: render splitter parameters from the component schema --- .../components/models/CreateSessionSteps.jsx | 13 +- .../components/models/SessionInfoContent.jsx | 35 +- .../modelSession/PrepareDatasetStep.jsx | 117 ++--- .../models/modelSession/SplitDatasetRows.jsx | 433 +++++------------- .../utils/i18n/locales/de/experiments.json | 4 - .../utils/i18n/locales/en/experiments.json | 4 - .../utils/i18n/locales/es/experiments.json | 4 - .../utils/i18n/locales/pt/experiments.json | 4 - .../utils/i18n/locales/zh/experiments.json | 4 - DashAI/front/src/utils/splitsPayload.js | 50 ++ 10 files changed, 243 insertions(+), 425 deletions(-) diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 0a838ffe7..2a4dc4212 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -16,6 +16,7 @@ import { import { useTranslation } from "react-i18next"; import { useModels } from "./ModelsContext"; import StepperNavigationFooter from "../shared/StepperNavigationFooter"; +import { hasPartition } from "../../utils/splitsPayload"; function CreateSessionSteps({ backHome, @@ -164,15 +165,9 @@ function CreateSessionSteps({ console.warn("Could not fetch metrics:", error); } - const hasTrain = - (newExp.splits.train !== undefined && newExp.splits.train !== 0) || - evaluationStrategy === "CrossValidationEvaluationStrategy"; - const hasValidation = - newExp.splits.validation !== undefined && - newExp.splits.validation !== 0; - const hasTest = - (newExp.splits.test !== undefined && newExp.splits.test !== 0) || - evaluationStrategy === "CrossValidationEvaluationStrategy"; + const hasTrain = hasPartition(newExp.splits, "train"); + const hasValidation = hasPartition(newExp.splits, "validation"); + const hasTest = hasPartition(newExp.splits, "test"); let effectiveName = sessionName; let response; diff --git a/DashAI/front/src/components/models/SessionInfoContent.jsx b/DashAI/front/src/components/models/SessionInfoContent.jsx index a4b00e71b..3d97764fe 100644 --- a/DashAI/front/src/components/models/SessionInfoContent.jsx +++ b/DashAI/front/src/components/models/SessionInfoContent.jsx @@ -9,6 +9,7 @@ const SPLIT_TYPE_LABEL_KEYS = { random: "experiments:label.random", manual: "experiments:label.manual", predefined: "experiments:label.predefined", + cv: "experiments:label.crossValidation", }; /** @@ -77,6 +78,11 @@ export default function SessionInfoContent({ t(SPLIT_TYPE_LABEL_KEYS[splits.splitType] || splits.splitType), ]); + // Sessions created before the splits payload followed the splitter schema + // stored the seed as "seed" and the manual indexes under the partition + // names, so both shapes are read here. + const seed = splits.random_state ?? splits.seed; + if (splits.splitType === "random") { configRows.push( [t("common:train"), splits.train], @@ -84,13 +90,34 @@ export default function SessionInfoContent({ [t("common:test"), splits.test], [t("experiments:label.shuffle"), yesNo(splits.shuffle)], [t("experiments:label.stratify"), yesNo(splits.stratify)], - [t("experiments:label.seed"), splits.seed], + [t("experiments:label.seed"), seed], ); + } else if (splits.splitType === "cv") { + if (splits.n_splits !== undefined) { + configRows.push([t("experiments:label.numFolds"), splits.n_splits]); + } + if (splits.n_repeats !== undefined) { + configRows.push([t("experiments:label.numRepeats"), splits.n_repeats]); + } + if (splits.group_column) { + configRows.push([ + t("experiments:label.groupColumn"), + splits.group_column, + ]); + } + configRows.push([t("experiments:label.seed"), seed]); } else { + const indexes = splits.splitted_indexes || {}; configRows.push( - [t("common:train"), (splits.train || []).length], - [t("common:validation"), (splits.validation || []).length], - [t("common:test"), (splits.test || []).length], + [ + t("common:train"), + (indexes.train_indexes ?? splits.train ?? []).length, + ], + [ + t("common:validation"), + (indexes.val_indexes ?? splits.validation ?? []).length, + ], + [t("common:test"), (indexes.test_indexes ?? splits.test ?? []).length], ); } } diff --git a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx index a063f9ef7..07cde368d 100644 --- a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx @@ -22,6 +22,11 @@ import { getColorByColumnType } from "../../../utils"; import { useTranslation } from "react-i18next"; import { Trans } from "react-i18next"; import { useModels } from "../ModelsContext"; +import { + buildSplitsPayload, + resolveSplitterName, + SPLIT_TYPES, +} from "../../../utils/splitsPayload"; /** * Step of the experiment modal: Set the input and output columns to use for clasification * and the splits for training, validation and testing. @@ -66,14 +71,14 @@ function PrepareDatasetStep({ // banner doesn't flash red while columns are still being auto-selected or a // check is in flight, only once a real valid/invalid result is known. const [validationPending, setValidationPending] = useState(true); - const [shuffle, setShuffle] = useState(true); - const [stratify, setStratify] = useState(false); - const [seed, setSeed] = useState(42); + + // Values submitted by the schema generated splitter form, and whether that + // form currently reports a validation error. + const [splitterParams, setSplitterParams] = useState(null); + const [paramsError, setParamsError] = useState(false); // Cross-Validation configuration states const [cvType, setCvType] = useState(null); - const [numFolds, setNumFolds] = useState(5); - const [numRepeats, setNumRepeats] = useState(2); const [groupColumn, setGroupColumn] = useState(""); const defaultParitionsIndex = { @@ -81,25 +86,11 @@ function PrepareDatasetStep({ validation: [], test: [], }; - const defaultPartitionsPercentage = { - train: 0.6, - validation: 0.2, - test: 0.2, - }; - const [datasetPartitionsIndex, setDatasetPartitionsIndex] = useState({}); const [rowsPartitionsIndex, setRowsPartitionsIndex] = useState( defaultParitionsIndex, ); - const [rowsPartitionsPercentage, setRowsPartitionsPercentage] = useState( - defaultPartitionsPercentage, - ); - const SPLIT_TYPES = { - RANDOM: "random", - MANUAL: "manual", - PREDEFINED: "predefined", - }; const [splitType, setSplitType] = useState(""); const [splitsReady, setSplitsReady] = useState(false); @@ -258,41 +249,27 @@ function PrepareDatasetStep({ evaluation_strategy: evaluationStrategy, }; - if (evaluationStrategy === "HoldoutEvaluationStrategy") { - if (splitType === SPLIT_TYPES.MANUAL) { - updatedExpData.splits = { - ...rowsPartitionsIndex, - splitter_name: "HoldoutSplitter", - splitType: splitType, - }; - } else if (splitType === SPLIT_TYPES.RANDOM) { - updatedExpData.splits = { - ...rowsPartitionsPercentage, - shuffle: shuffle, - stratify: stratify, - seed: seed === "" || seed == null ? 42 : Number(seed), - splitter_name: "HoldoutSplitter", - splitType: splitType, - }; - } else if (splitType === SPLIT_TYPES.PREDEFINED) { - updatedExpData.splits = { - ...datasetPartitionsIndex, - splitter_name: "HoldoutSplitter", - splitType: splitType, - }; - } - } else if (evaluationStrategy === "CrossValidationEvaluationStrategy") { - const cvSchemaProperties = cvType.schema?.properties || {}; - updatedExpData.splits = { - splitter_name: cvType.name, - seed: seed === "" || seed == null ? 42 : Number(seed), - ...(cvSchemaProperties.n_splits ? { n_splits: numFolds } : {}), - ...(cvSchemaProperties.n_repeats ? { n_repeats: numRepeats } : {}), - ...(cvSchemaProperties.group_column - ? { group_column: groupColumn } - : {}), - ...(cvSchemaProperties.shuffle ? { shuffle: shuffle } : {}), - }; + const splitterName = resolveSplitterName(evaluationStrategy, cvType); + if (splitterName) { + updatedExpData.splits = buildSplitsPayload({ + splitterName, + splitType: + evaluationStrategy === "HoldoutEvaluationStrategy" + ? splitType + : SPLIT_TYPES.CV, + params: { + ...(splitterParams ?? {}), + // The group column select is rendered by hand, so its value is not + // part of the generated form's values. + ...(cvType?.schema?.properties?.group_column + ? { group_column: groupColumn } + : {}), + }, + indexes: + splitType === SPLIT_TYPES.PREDEFINED + ? datasetPartitionsIndex + : rowsPartitionsIndex, + }); } setNewExp(updatedExpData); @@ -340,16 +317,14 @@ function PrepareDatasetStep({ splitsReady, columnsAreValid, splitType, - shuffle, - stratify, - seed, + splitterParams, inputColumnNames, outputColumnNames, cvType, - numFolds, - numRepeats, groupColumn, evaluationStrategy, + rowsPartitionsIndex, + datasetPartitionsIndex, ]); useEffect(() => { @@ -382,26 +357,18 @@ function PrepareDatasetStep({ datasetInfo={datasetInfo} rowsPartitionsIndex={rowsPartitionsIndex} setRowsPartitionsIndex={setRowsPartitionsIndex} - rowsPartitionsPercentage={rowsPartitionsPercentage} - setRowsPartitionsPercentage={setRowsPartitionsPercentage} setSplitsReady={setSplitsReady} splitType={splitType} setSplitType={setSplitType} SPLIT_TYPES={SPLIT_TYPES} - shuffle={shuffle} - setShuffle={setShuffle} - stratify={stratify} - setStratify={setStratify} - seed={seed} - setSeed={setSeed} + splitterParams={splitterParams} + setSplitterParams={setSplitterParams} + paramsError={paramsError} + setParamsError={setParamsError} evaluationStrategy={evaluationStrategy} setEvaluationStrategy={setEvaluationStrategy} cvType={cvType} setCvType={setCvType} - numFolds={numFolds} - setNumFolds={setNumFolds} - numRepeats={numRepeats} - setNumRepeats={setNumRepeats} groupColumn={groupColumn} setGroupColumn={setGroupColumn} inputColumnNames={inputColumnNames} @@ -413,15 +380,11 @@ function PrepareDatasetStep({ infoLoading, datasetInfo, rowsPartitionsIndex, - rowsPartitionsPercentage, splitType, - shuffle, - stratify, - seed, + splitterParams, + paramsError, evaluationStrategy, cvType, - numFolds, - numRepeats, groupColumn, inputColumnNames, ]); diff --git a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx index c7110e4bc..6023b47dc 100644 --- a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx +++ b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import PropTypes from "prop-types"; import { parseRangeToIndex } from "../../../utils/parseRange"; import { @@ -17,14 +17,15 @@ import { FormControl, InputLabel, } from "@mui/material"; -import BooleanInput from "../../configurableObject/Inputs/BooleanInput"; -import FormSchemaFieldCard, { - DescriptionBlock, -} from "../../shared/FormSchemaFieldCard"; +import { DescriptionBlock } from "../../shared/FormSchemaFieldCard"; +import FormSchema from "../../shared/FormSchema"; +import FormSchemaLayout from "../../shared/FormSchemaLayout"; +import { resolveSplitterName } from "../../../utils/splitsPayload"; import { useTranslation } from "react-i18next"; import { SplitscreenOutlined } from "@mui/icons-material"; import { useSnackbar } from "notistack"; import { getComponents } from "../../../api/component"; +import useSchema from "../../../hooks/useSchema"; /** * Splits card shell — same Paper/header visual as FormSchemaFieldCard but WITHOUT @@ -74,26 +75,18 @@ function SplitDatasetRows({ datasetInfo, rowsPartitionsIndex, setRowsPartitionsIndex, - rowsPartitionsPercentage, - setRowsPartitionsPercentage, setSplitsReady, splitType, setSplitType, SPLIT_TYPES, - shuffle, - setShuffle, - stratify, - setStratify, - seed, - setSeed, + splitterParams, + setSplitterParams, + paramsError, + setParamsError, evaluationStrategy, setEvaluationStrategy, cvType, setCvType, - numFolds, - setNumFolds, - numRepeats, - setNumRepeats, groupColumn, setGroupColumn, inputColumnNames, @@ -101,6 +94,10 @@ function SplitDatasetRows({ }) { const { t } = useTranslation(["experiments", "common"]); const { enqueueSnackbar } = useSnackbar(); + // The generated form reports its values only when the user edits a field, so + // the schema placeholders are pushed up once per splitter. Without this a + // session created without touching the form would carry no parameters. + const seededSplitter = useRef(null); const totalRows = datasetInfo.total_rows; const trainDatasetPercentage = (datasetInfo.train_size / totalRows).toFixed( @@ -119,15 +116,62 @@ function SplitDatasetRows({ const checkSplit = (train, validation, test) => Math.abs(train + validation + test - 1) < 0.0001; - const [randomSplitError, setRandomSplitError] = useState(false); - const [randomSplitErrorText, setRandomSplitErrorText] = useState(""); + // The splitter's own parameters come from the schema generated form; only the + // rules the schema cannot express are checked here. + const splitterName = resolveSplitterName(evaluationStrategy, cvType); + const isIndexMode = + splitType === SPLIT_TYPES.MANUAL || splitType === SPLIT_TYPES.PREDEFINED; + const params = splitterParams ?? {}; + const proportionsAreNumbers = ["train", "validation", "test"].every( + (key) => typeof params[key] === "number", + ); + const proportionsSumToOne = + proportionsAreNumbers && + checkSplit(params.train, params.validation, params.test); + const trainIsEmpty = params.train === 0; + const folds = params.n_splits; + const tooManyFolds = + typeof folds === "number" && folds > (datasetInfo.total_rows ?? 0); + // group_column is rendered by hand because its options are the dataset's own + // columns, so the generated form never validates it. + const omittedFields = isIndexMode + ? ["group_column", "train", "test", "validation"] + : ["group_column"]; + + const { defaultValues: splitterDefaults } = useSchema({ + modelName: splitterName, + }); + + useEffect(() => { + if (!splitterName || seededSplitter.current === splitterName) return; + if (!splitterDefaults || Object.keys(splitterDefaults).length === 0) return; + seededSplitter.current = splitterName; + setSplitterParams(splitterDefaults); + }, [splitterName, splitterDefaults]); + + // Rules that span several fields, which a per field schema cannot express. + let splitsMessage = ""; + if ( + evaluationStrategy === "HoldoutEvaluationStrategy" && + splitType === SPLIT_TYPES.RANDOM + ) { + if (trainIsEmpty) { + splitsMessage = t("experiments:error.trainSplitMustBeGreaterThanZero"); + } else if (proportionsAreNumbers && !proportionsSumToOne) { + splitsMessage = t("experiments:error.splitsMustSumToOne"); + } + } else if ( + evaluationStrategy === "CrossValidationEvaluationStrategy" && + tooManyFolds + ) { + splitsMessage = t("experiments:error.foldsMustBeLessThanDatasetSize", { + datasetSize: datasetInfo.total_rows, + }); + } + const [manualSplitError, setManualSplitError] = useState(false); const [manualSplitErrorText, setManualSplitErrorText] = useState(""); - const [cvFoldError, setCvFoldError] = useState(false); - const [cvFoldErrorText, setCvFoldErrorText] = useState(""); - const [cvRepeatError, setCvRepeatError] = useState(false); - const [cvRepeatErrorText, setCvRepeatErrorText] = useState(""); const [groupColumnError, setGroupColumnError] = useState(true); const [allowedCvTypes, setAllowedCvTypes] = useState([]); @@ -163,27 +207,6 @@ function SplitDatasetRows({ if (newType === SPLIT_TYPES.PREDEFINED) { setSplitsReady(true); } - if (newType === SPLIT_TYPES.RANDOM) { - const newSplit = { train: 0.6, test: 0.2, validation: 0.2 }; - setRowsPartitionsPercentage(newSplit); - const hasZero = newSplit.train === 0; - const sumsToOne = checkSplit( - newSplit.train, - newSplit.validation, - newSplit.test, - ); - if (hasZero) { - setRandomSplitErrorText( - t("experiments:error.trainSplitMustBeGreaterThanZero"), - ); - setRandomSplitError(true); - } else if (!sumsToOne) { - setRandomSplitErrorText(t("experiments:error.splitsMustSumToOne")); - setRandomSplitError(true); - } else { - setRandomSplitError(false); - } - } if (newType === SPLIT_TYPES.MANUAL) { const newIndex = { train: [], test: [], validation: [] }; setRowsPartitionsIndex(newIndex); @@ -198,62 +221,24 @@ function SplitDatasetRows({ const value = event.target.value; const id = event.target.id; - if (splitType === SPLIT_TYPES.MANUAL) { - try { - const rowsIndex = parseRangeToIndex(value, totalRows); - const updatedIndex = { ...rowsPartitionsIndex, [id]: rowsIndex }; - setRowsPartitionsIndex(updatedIndex); - if (updatedIndex.train.length === 0) { - setManualSplitErrorText( - t("experiments:error.trainSplitMustHaveAtLeastOneRow"), - ); - setManualSplitError(true); - } else { - setManualSplitError(false); - } - } catch (error) { - setManualSplitErrorText(error.message); - setManualSplitError(true); - } - } else { - const numValue = parseFloat(value) || 0; - const newSplit = { ...rowsPartitionsPercentage, [id]: numValue }; - setRowsPartitionsPercentage(newSplit); - const hasZero = newSplit.train === 0; - const sumsToOne = checkSplit( - newSplit.train, - newSplit.validation, - newSplit.test, - ); - if (hasZero) { - setRandomSplitErrorText( - t("experiments:error.trainSplitMustBeGreaterThanZero"), + try { + const rowsIndex = parseRangeToIndex(value, totalRows); + const updatedIndex = { ...rowsPartitionsIndex, [id]: rowsIndex }; + setRowsPartitionsIndex(updatedIndex); + if (updatedIndex.train.length === 0) { + setManualSplitErrorText( + t("experiments:error.trainSplitMustHaveAtLeastOneRow"), ); - setRandomSplitError(true); - } else if (!sumsToOne) { - setRandomSplitErrorText(t("experiments:error.splitsMustSumToOne")); - setRandomSplitError(true); + setManualSplitError(true); } else { - setRandomSplitError(false); + setManualSplitError(false); } + } catch (error) { + setManualSplitErrorText(error.message); + setManualSplitError(true); } }; - const handleShuffleChange = (value) => { - setShuffle(value); - if (!value) setStratify(false); - }; - - const handleStratifyChange = (value) => { - if (shuffle) setStratify(value); - else setStratify(false); - }; - - const handleSeedChange = (event) => { - const value = event.target.value === "" ? "" : Number(event.target.value); - setSeed(value); - }; - const handleGroupColumnChange = (event) => { const value = event.target.value; setGroupColumn(value); @@ -264,53 +249,6 @@ function SplitDatasetRows({ } }; - const handleNumFoldsChange = (event) => { - const value = event.target.value === "" ? "" : Number(event.target.value); - setNumFolds(value); - if (value !== "" && value > 1 && value <= datasetInfo.total_rows) { - setCvFoldError(false); - setCvFoldErrorText(""); - return; - } - - if (value === "") { - setCvFoldErrorText(t("experiments:error.foldsMustBeAnInteger")); - } else if (value <= 1) { - setCvFoldErrorText(t("experiments:error.foldsMustBeGreaterThanOne")); - } else if (value > datasetInfo.total_rows) { - setCvFoldErrorText( - t("experiments:error.foldsMustBeLessThanDatasetSize", { - datasetSize: datasetInfo.total_rows, - }), - ); - } - setCvFoldError(true); - return; - }; - - const handleOnBlurSeed = (event) => { - if (event.target.value === "") { - setSeed(42); - } - }; - - const handleNumRepeatsChange = (event) => { - const value = event.target.value === "" ? "" : Number(event.target.value); - setNumRepeats(value); - if (value !== "" && value > 1) { - setCvRepeatError(false); - setCvRepeatErrorText(""); - return; - } - if (value === "") { - setCvRepeatErrorText(t("experiments:error.repeatsMustBeGreaterThanOne")); - } else if (value <= 1) { - setCvRepeatErrorText(t("experiments:error.repeatsMustBeGreaterThanOne")); - } - setCvRepeatError(true); - return; - }; - useEffect(() => { if (hasPredefinedSplits) { setSplitType(SPLIT_TYPES.PREDEFINED); @@ -320,6 +258,10 @@ function SplitDatasetRows({ }, [hasPredefinedSplits]); useEffect(() => { + if (paramsError) { + setSplitsReady(false); + return; + } if (evaluationStrategy === "HoldoutEvaluationStrategy") { if (splitType === SPLIT_TYPES.PREDEFINED) { setSplitsReady(true); @@ -331,32 +273,31 @@ function SplitDatasetRows({ setSplitsReady(true); } else if ( splitType === SPLIT_TYPES.RANDOM && - !randomSplitError && - rowsPartitionsPercentage.train > 0 + proportionsSumToOne && + !trainIsEmpty ) { setSplitsReady(true); } else { setSplitsReady(false); } } else if (evaluationStrategy === "CrossValidationEvaluationStrategy") { - if (!cvFoldError && !cvRepeatError && !groupColumnError) { - setSplitsReady(true); - } else { - setSplitsReady(false); - } + setSplitsReady( + Boolean(splitterName) && !tooManyFolds && !groupColumnError, + ); } else { setSplitsReady(false); } }, [ rowsPartitionsIndex, - rowsPartitionsPercentage, - randomSplitError, manualSplitError, splitType, evaluationStrategy, - cvFoldError, - cvRepeatError, + paramsError, + proportionsSumToOne, + trainIsEmpty, + tooManyFolds, groupColumnError, + splitterName, ]); const splitOptions = [ @@ -510,85 +451,6 @@ function SplitDatasetRows({ )} - {/* Random */} - {splitType === SPLIT_TYPES.RANDOM && ( - <> - - - {splitFields.map(({ id, label }) => ( - - - - ))} - - - - - - - - - - - - - - - - )} - {/* Manual */} {splitType === SPLIT_TYPES.MANUAL && ( - {/* Number of Folds */} - {Boolean(cvType?.schema?.properties?.n_splits) && ( - - - - )} - - {/* Number of Repeats */} - {Boolean(cvType?.schema?.properties?.n_repeats) && ( - - - - )} - {/* Group Column */} {Boolean(cvType?.schema?.properties?.group_column) && ( )} + + )} - {/* Shuffle */} - {Boolean(cvType?.schema?.properties?.shuffle) && ( - - - - )} - - {/* Seed */} - - + + - + + {splitsMessage && ( + + {splitsMessage} + + )} )} @@ -744,26 +555,18 @@ SplitDatasetRows.propTypes = { datasetInfo: PropTypes.object.isRequired, rowsPartitionsIndex: PropTypes.object.isRequired, setRowsPartitionsIndex: PropTypes.func.isRequired, - rowsPartitionsPercentage: PropTypes.object.isRequired, - setRowsPartitionsPercentage: PropTypes.func.isRequired, setSplitsReady: PropTypes.func.isRequired, splitType: PropTypes.string.isRequired, setSplitType: PropTypes.func.isRequired, SPLIT_TYPES: PropTypes.object.isRequired, - shuffle: PropTypes.bool.isRequired, - setShuffle: PropTypes.func.isRequired, - stratify: PropTypes.bool.isRequired, - setStratify: PropTypes.func.isRequired, - seed: PropTypes.oneOfType([PropTypes.number, PropTypes.string]).isRequired, - setSeed: PropTypes.func.isRequired, + splitterParams: PropTypes.object, + setSplitterParams: PropTypes.func.isRequired, + paramsError: PropTypes.bool, + setParamsError: PropTypes.func.isRequired, evaluationStrategy: PropTypes.string.isRequired, setEvaluationStrategy: PropTypes.func.isRequired, - cvType: PropTypes.string.isRequired, + cvType: PropTypes.object, setCvType: PropTypes.func.isRequired, - numFolds: PropTypes.number.isRequired, - setNumFolds: PropTypes.func.isRequired, - numRepeats: PropTypes.number.isRequired, - setNumRepeats: PropTypes.func.isRequired, groupColumn: PropTypes.string, setGroupColumn: PropTypes.func.isRequired, inputColumnNames: PropTypes.arrayOf(PropTypes.string), diff --git a/DashAI/front/src/utils/i18n/locales/de/experiments.json b/DashAI/front/src/utils/i18n/locales/de/experiments.json index dd192965d..ce39fb263 100644 --- a/DashAI/front/src/utils/i18n/locales/de/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/de/experiments.json @@ -28,10 +28,7 @@ "taskRequirementsNotFound": "Aufgabenanforderungen für {{taskName}} nicht gefunden", "trainSplitMustBeGreaterThanZero": "Trainingsaufteilung muss größer als null sein", "trainSplitMustHaveAtLeastOneRow": "Trainingsaufteilung muss mindestens eine Zeile haben", - "foldsMustBeGreaterThanOne": "Die Anzahl der Folds muss größer als 1 sein", "foldsMustBeLessThanDatasetSize": "Die Anzahl der Folds muss kleiner oder gleich der Anzahl der Stichproben im Datensatz sein ({{datasetSize}})", - "foldsMustBeAnInteger": "Die Anzahl der Folds muss eine ganze Zahl sein", - "repeatsMustBeGreaterThanOne": "Die Anzahl der Wiederholungen muss größer als 1 sein", "groupColumnMustBeSelected": "Es muss eine Spalte zum Gruppieren ausgewählt werden" }, "label": { @@ -48,7 +45,6 @@ "datasetOutputColumnRequirements": "<0>Die Ausgabespalten müssen folgende Typen haben<1><2>, und eine Kardinalität von <3>{{cardinality}}.", "duration": "Dauer", "endTime": "Endzeit", - "enterSeedValue": "Seed-Wert eingeben (Standard: 42)", "experimentName": "Experimentname", "experimentsModuleTitle": "Experiment-Modul", "metricToOptimize": "Zu optimierende Metrik", diff --git a/DashAI/front/src/utils/i18n/locales/en/experiments.json b/DashAI/front/src/utils/i18n/locales/en/experiments.json index b097e20f5..f3c40dec0 100644 --- a/DashAI/front/src/utils/i18n/locales/en/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/en/experiments.json @@ -28,10 +28,7 @@ "taskRequirementsNotFound": "Task requirements for {{taskName}} not found", "trainSplitMustBeGreaterThanZero": "Train split must be greater than zero", "trainSplitMustHaveAtLeastOneRow": "Train split must have at least one row", - "foldsMustBeGreaterThanOne": "Number of folds must be greater than 1", "foldsMustBeLessThanDatasetSize": "Number of folds must be less than or equal to the number of samples in the dataset ({{datasetSize}})", - "foldsMustBeAnInteger": "Number of folds must be an integer", - "repeatsMustBeGreaterThanOne": "Number of repeats must be greater than 1", "groupColumnMustBeSelected": "A group column must be selected" }, "label": { @@ -48,7 +45,6 @@ "datasetOutputColumnRequirements": "<0>The output columns must be of the types<1><2>, and they should have a cardinality of <3>{{cardinality}}.", "duration": "Duration", "endTime": "End Time", - "enterSeedValue": "Enter a seed value (default: 42)", "experimentName": "Experiment Name", "experimentsModuleTitle": "Experiments Module", "metricToOptimize": "Metric to Optimize", diff --git a/DashAI/front/src/utils/i18n/locales/es/experiments.json b/DashAI/front/src/utils/i18n/locales/es/experiments.json index 5bb2d040a..c2ff88b13 100644 --- a/DashAI/front/src/utils/i18n/locales/es/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/es/experiments.json @@ -28,10 +28,7 @@ "taskRequirementsNotFound": "No se encontraron requisitos de tarea para {{taskName}}", "trainSplitMustBeGreaterThanZero": "La división de entrenamiento debe ser mayor que cero", "trainSplitMustHaveAtLeastOneRow": "La división de entrenamiento debe tener al menos una fila", - "foldsMustBeGreaterThanOne": "El número de pliegues debe ser mayor que 1", "foldsMustBeLessThanDatasetSize": "El número de pliegues debe ser menor o igual al número de muestras en el dataset ({{datasetSize}})", - "foldsMustBeAnInteger": "El número de pliegues debe ser un entero", - "repeatsMustBeGreaterThanOne": "El número de repeticiones debe ser mayor que 1", "groupColumnMustBeSelected": "Debe seleccionarse una columna para agrupar" }, "label": { @@ -48,7 +45,6 @@ "datasetOutputColumnRequirements": "<0>Las columnas de salida deben ser de los tipos<1><2>, y deben tener una cardinalidad de <3>{{cardinality}}.", "duration": "Duración", "endTime": "Hora de Finalización", - "enterSeedValue": "Ingrese un valor de semilla (predeterminado: 42)", "experimentName": "Nombre del Experimento", "experimentsModuleTitle": "Módulo de Experimentos", "metricToOptimize": "Métrica a Optimizar", diff --git a/DashAI/front/src/utils/i18n/locales/pt/experiments.json b/DashAI/front/src/utils/i18n/locales/pt/experiments.json index 55c519883..497b678fd 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/pt/experiments.json @@ -28,10 +28,7 @@ "taskRequirementsNotFound": "Nenhum requisito de tarefa encontrado para {{taskName}}", "trainSplitMustBeGreaterThanZero": "A divisão de treinamento deve ser maior que zero", "trainSplitMustHaveAtLeastOneRow": "A divisão de treinamento deve ter pelo menos uma linha", - "foldsMustBeGreaterThanOne": "O número de dobras deve ser maior que 1", "foldsMustBeLessThanDatasetSize": "O número de dobras deve ser menor ou igual ao número de amostras no conjunto de dados ({{datasetSize}})", - "foldsMustBeAnInteger": "O número de dobras deve ser um número inteiro", - "repeatsMustBeGreaterThanOne": "O número de repetições deve ser maior que 1", "groupColumnMustBeSelected": "Uma coluna para agrupamento deve ser selecionada" }, "label": { @@ -48,7 +45,6 @@ "datasetOutputColumnRequirements": "<0>As colunas de saída devem ser dos tipos<1><2>, e devem ter uma cardinalidade de <3>{{cardinality}}.", "duration": "Duração", "endTime": "Hora de Conclusão", - "enterSeedValue": "Insira um valor de semente (padrão: 42)", "experimentName": "Nome do Experimento", "experimentsModuleTitle": "Módulo de Experimentos", "metricToOptimize": "Métrica a Otimizar", diff --git a/DashAI/front/src/utils/i18n/locales/zh/experiments.json b/DashAI/front/src/utils/i18n/locales/zh/experiments.json index 9ac9c330d..7ff95c092 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/zh/experiments.json @@ -28,10 +28,7 @@ "taskRequirementsNotFound": "未找到 {{taskName}} 的任务需求", "trainSplitMustBeGreaterThanZero": "训练集比例必须大于零", "trainSplitMustHaveAtLeastOneRow": "训练集至少需要一行数据", - "foldsMustBeGreaterThanOne": "折数必须大于 1", "foldsMustBeLessThanDatasetSize": "折数必须小于或等于数据集中的样本数({{datasetSize}})", - "foldsMustBeAnInteger": "折数必须为整数", - "repeatsMustBeGreaterThanOne": "重复次数必须大于 1", "groupColumnMustBeSelected": "必须选择一个用于分组的列" }, "label": { @@ -48,7 +45,6 @@ "datasetOutputColumnRequirements": "<0>输出列的类型必须为<1><2>,且基数应为<3>{{cardinality}}。", "duration": "时长", "endTime": "结束时间", - "enterSeedValue": "输入随机种子值(默认值:42)", "experimentName": "实验名称", "experimentsModuleTitle": "实验模块", "metricToOptimize": "待优化指标", diff --git a/DashAI/front/src/utils/splitsPayload.js b/DashAI/front/src/utils/splitsPayload.js index 2b1b847fe..2bf39ebb1 100644 --- a/DashAI/front/src/utils/splitsPayload.js +++ b/DashAI/front/src/utils/splitsPayload.js @@ -5,6 +5,9 @@ export const SPLIT_TYPES = { CV: "cv", }; +export const HOLDOUT_STRATEGY = "HoldoutEvaluationStrategy"; +export const CV_STRATEGY = "CrossValidationEvaluationStrategy"; + // Proportions describe a random split only. Manual and predefined splits carry // the row indexes instead, and the backend splitter picks its index branch when // no proportion is present, so sending both would be ambiguous. @@ -12,6 +15,12 @@ const PROPORTION_KEYS = ["train", "test", "validation"]; const INDEX_MODES = [SPLIT_TYPES.MANUAL, SPLIT_TYPES.PREDEFINED]; +const PARTITION_INDEX_KEYS = { + train: "train_indexes", + validation: "val_indexes", + test: "test_indexes", +}; + /** * Build the splits payload stored on a model session. * @@ -52,3 +61,44 @@ export const buildSplitsPayload = ({ return payload; }; + +/** + * Resolve the registry name of the splitter a session configuration uses. + * + * Holdout has a single splitter; cross-validation picks one from the registry. + * + * @param {string} evaluationStrategy the selected evaluation strategy + * @param {object} cvType the splitter component selected for cross-validation + * @returns {string|null} the splitter name, or null when none is resolved yet + */ +export const resolveSplitterName = (evaluationStrategy, cvType) => { + if (evaluationStrategy === HOLDOUT_STRATEGY) return "HoldoutSplitter"; + if (evaluationStrategy === CV_STRATEGY) return cvType?.name ?? null; + return null; +}; + +/** + * Report whether a splits payload gives a partition any rows. + * + * Cross-validation folds every row into train and test, and never builds a + * validation partition. Index modes carry row indexes, random splits carry + * proportions, and payloads written before the splits followed the splitter + * schema carry the indexes under the partition names. + * + * @param {object} splits a splits payload + * @param {string} partition one of "train", "validation", "test" + * @returns {boolean} true when the partition receives rows + */ +export const hasPartition = (splits, partition) => { + if (!splits) return false; + if (splits.splitType === SPLIT_TYPES.CV) return partition !== "validation"; + + const indexes = splits.splitted_indexes; + if (indexes) { + return (indexes[PARTITION_INDEX_KEYS[partition]] ?? []).length > 0; + } + + const value = splits[partition]; + if (Array.isArray(value)) return value.length > 0; + return typeof value === "number" && value !== 0; +}; From 53b59fa0bb109a7aaa97ba3bfd73abbf1ff104bf Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 10:08:30 -0400 Subject: [PATCH 10/16] test: cover schema splits payloads end to end --- tests/back/api/test_jobs.py | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/back/api/test_jobs.py b/tests/back/api/test_jobs.py index 9dba816d7..57823378d 100644 --- a/tests/back/api/test_jobs.py +++ b/tests/back/api/test_jobs.py @@ -319,6 +319,140 @@ def test_job_with_wrong_run(client: TestClient): assert response.status_code == 500, response.text +def _run_with_splits( + client: TestClient, model_session_id: int, splits: dict, run_name: str +): + """Train a run over a session temporarily configured with given splits. + + Parameters + ---------- + client : TestClient + The API test client. + model_session_id : int + Session whose splits are swapped for the duration of the call. + splits : dict + The splits payload to store on the session. + run_name : str + Name for the run created over that session. + + Returns + ------- + tuple + (job status string, split indexes stored on the run). + """ + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + original_splits = model_session.splits + model_session.splits = json.dumps(splits) + db.commit() + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "DummyModel", + "name": run_name, + "parameters": {}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Run over a specific splits payload", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 201, response.text + run_id = response.json()["id"] + + try: + response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert response.status_code == 201, response.text + job_id = response.json()["id"] + + response = client.get(f"/api/v1/job/status/{job_id}") + assert response.status_code == 200, response.text + status = response.json()["status"] + + with session_factory() as db: + run = db.get(Run, run_id) + split_indexes = json.loads(run.split_indexes) if run.split_indexes else None + + return status, split_indexes + finally: + with session_factory() as db: + db.get(ModelSession, model_session_id).splits = original_splits + db.commit() + client.delete(f"/api/v1/run/{run_id}") + + +def test_manual_splits_from_the_splitter_schema_train( + client: TestClient, model_session_id: int +): + """The payload the wizard writes for a manual split trains as given.""" + status, split_indexes = _run_with_splits( + client, + model_session_id, + { + "splitter_name": "HoldoutSplitter", + "splitType": "manual", + "splitted_indexes": { + "train_indexes": [0, 1, 2, 3, 4, 5], + "test_indexes": [6, 7], + "val_indexes": [8, 9], + }, + "stratify": False, + "shuffle": True, + "random_state": 42, + }, + "SchemaManualSplitRun", + ) + + assert status == "finished" + assert split_indexes == { + "train_indexes": [0, 1, 2, 3, 4, 5], + "test_indexes": [6, 7], + "val_indexes": [8, 9], + } + + +def test_random_splits_honor_the_random_state( + client: TestClient, model_session_id: int +): + """The seed reaches the splitter, so two seeds give two different splits.""" + base = { + "splitter_name": "HoldoutSplitter", + "splitType": "random", + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": True, + } + + first_status, first = _run_with_splits( + client, model_session_id, {**base, "random_state": 1}, "SeedOneRun" + ) + other_status, other = _run_with_splits( + client, model_session_id, {**base, "random_state": 99}, "SeedNinetyNineRun" + ) + + assert first_status == "finished" + assert other_status == "finished" + assert first["train_indexes"] != other["train_indexes"] + + def test_legacy_manual_splits_still_train(client: TestClient, model_session_id: int): """A session stored before the splits payload followed the splitter schema. From c26dfbdf080240faa56e4ebb7ec4c469d6bc96aa Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 10:11:22 -0400 Subject: [PATCH 11/16] refactor: drop dead imports from the splits step --- .../models/modelSession/SplitDatasetRows.jsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx index 6023b47dc..dd4dafb55 100644 --- a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx +++ b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from "react"; +import React, { useEffect, useMemo, useRef, useState } from "react"; import PropTypes from "prop-types"; import { parseRangeToIndex } from "../../../utils/parseRange"; import { @@ -11,18 +11,15 @@ import { ToggleButton, ToggleButtonGroup, Typography, - Button, Select, MenuItem, FormControl, - InputLabel, } from "@mui/material"; import { DescriptionBlock } from "../../shared/FormSchemaFieldCard"; import FormSchema from "../../shared/FormSchema"; import FormSchemaLayout from "../../shared/FormSchemaLayout"; import { resolveSplitterName } from "../../../utils/splitsPayload"; import { useTranslation } from "react-i18next"; -import { SplitscreenOutlined } from "@mui/icons-material"; import { useSnackbar } from "notistack"; import { getComponents } from "../../../api/component"; import useSchema from "../../../hooks/useSchema"; @@ -134,9 +131,14 @@ function SplitDatasetRows({ typeof folds === "number" && folds > (datasetInfo.total_rows ?? 0); // group_column is rendered by hand because its options are the dataset's own // columns, so the generated form never validates it. - const omittedFields = isIndexMode - ? ["group_column", "train", "test", "validation"] - : ["group_column"]; + // Memoized because the generated form memoizes its field list on this array. + const omittedFields = useMemo( + () => + isIndexMode + ? ["group_column", "train", "test", "validation"] + : ["group_column"], + [isIndexMode], + ); const { defaultValues: splitterDefaults } = useSchema({ modelName: splitterName, From 459e0ae7e5eff12d7a730d823184a2a5d49a2255 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 10:12:22 -0400 Subject: [PATCH 12/16] test: cover cross-validation splits validation --- .../test_model_session_splits_validation.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/back/api/test_model_session_splits_validation.py b/tests/back/api/test_model_session_splits_validation.py index 3820443a9..5017f97cc 100644 --- a/tests/back/api/test_model_session_splits_validation.py +++ b/tests/back/api/test_model_session_splits_validation.py @@ -87,6 +87,38 @@ def test_accepts_a_manual_split_without_proportions( assert response.status_code == 201, response.text +def test_accepts_a_cross_validation_payload(client: TestClient, dataset_1: Dataset): + splits = { + "splitter_name": "RepeatedKFoldSplitter", + "splitType": "cv", + "n_splits": 4, + "n_repeats": 3, + "random_state": 7, + } + body = _session_body(splits, dataset_1, "repeated cv session") + body["evaluation_strategy"] = "CrossValidationEvaluationStrategy" + + response = client.post("/api/v1/model-session/", json=body) + assert response.status_code == 201, response.text + + +def test_rejects_more_folds_than_the_schema_allows( + client: TestClient, dataset_1: Dataset +): + splits = { + "splitter_name": "KFoldSplitter", + "splitType": "cv", + "n_splits": 50, + "shuffle": True, + "random_state": 42, + } + body = _session_body(splits, dataset_1, "too many folds") + body["evaluation_strategy"] = "CrossValidationEvaluationStrategy" + + response = client.post("/api/v1/model-session/", json=body) + assert response.status_code == 422, response.text + + def test_accepts_a_legacy_seed_payload(client: TestClient, dataset_1: Dataset): splits = { "splitter_name": "HoldoutSplitter", From e84f0c10e3736d9908df99bfc985e869f732a36e Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 10:37:06 -0400 Subject: [PATCH 13/16] fix: keep the splits step ready after switching evaluation strategy --- .../modelSession/PrepareDatasetStep.jsx | 13 ++--- .../models/modelSession/SplitDatasetRows.jsx | 52 +++++++++++++------ 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx index 07cde368d..f94ceb24b 100644 --- a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx @@ -283,10 +283,13 @@ function PrepareDatasetStep({ } }, [inputColumnNames, outputColumnNames]); + // Column validity depends on the columns, the dataset and the task, never on + // the split configuration. Gating it on the splits being ready made every + // split change re-check the columns over HTTP and blank the requirements + // banner in the meantime. useEffect(() => { if ( columnsReady && - splitsReady && datasetInfo && datasetInfo.column_names && datasetInfo.column_names.length > 0 @@ -297,13 +300,7 @@ function PrepareDatasetStep({ setColumnsAreValid(false); setValidationPending(true); } - }, [ - columnsReady, - splitsReady, - inputColumnNames, - outputColumnNames, - datasetInfo, - ]); + }, [columnsReady, inputColumnNames, outputColumnNames, datasetInfo]); useEffect(() => { if (columnsAreValid && splitsReady && columnsReady) { diff --git a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx index dd4dafb55..6df6f0b18 100644 --- a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx +++ b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useRef, useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import PropTypes from "prop-types"; import { parseRangeToIndex } from "../../../utils/parseRange"; import { @@ -22,7 +22,6 @@ import { resolveSplitterName } from "../../../utils/splitsPayload"; import { useTranslation } from "react-i18next"; import { useSnackbar } from "notistack"; import { getComponents } from "../../../api/component"; -import useSchema from "../../../hooks/useSchema"; /** * Splits card shell — same Paper/header visual as FormSchemaFieldCard but WITHOUT @@ -91,11 +90,6 @@ function SplitDatasetRows({ }) { const { t } = useTranslation(["experiments", "common"]); const { enqueueSnackbar } = useSnackbar(); - // The generated form reports its values only when the user edits a field, so - // the schema placeholders are pushed up once per splitter. Without this a - // session created without touching the form would carry no parameters. - const seededSplitter = useRef(null); - const totalRows = datasetInfo.total_rows; const trainDatasetPercentage = (datasetInfo.train_size / totalRows).toFixed( 2, @@ -140,16 +134,42 @@ function SplitDatasetRows({ [isIndexMode], ); - const { defaultValues: splitterDefaults } = useSchema({ - modelName: splitterName, - }); - + // The generated form reports its values only once the user edits a field, so + // the schema placeholders are pushed up here: without them a session created + // without touching the form would carry no splitter parameters at all. + // + // The schema is fetched by name rather than read from the shared schema hook + // on purpose. That hook keeps the previous splitter's schema for one render + // after the name changes, and seeding from it left the parameters of the + // splitter that was selected a moment ago in place. useEffect(() => { - if (!splitterName || seededSplitter.current === splitterName) return; - if (!splitterDefaults || Object.keys(splitterDefaults).length === 0) return; - seededSplitter.current = splitterName; - setSplitterParams(splitterDefaults); - }, [splitterName, splitterDefaults]); + if (!splitterName) return undefined; + let cancelled = false; + + const seedSplitterParams = async () => { + try { + const component = await getComponents({ model: splitterName }); + if (cancelled) return; + const properties = component?.schema?.properties ?? {}; + const defaults = Object.fromEntries( + Object.entries(properties) + .filter(([, property]) => "placeholder" in property) + .map(([key, property]) => [key, property.placeholder]), + ); + if (Object.keys(defaults).length > 0) { + setSplitterParams(defaults); + } + } catch (error) { + console.error(`Error fetching the ${splitterName} schema`, error); + } + }; + + seedSplitterParams(); + + return () => { + cancelled = true; + }; + }, [splitterName]); // Rules that span several fields, which a per field schema cannot express. let splitsMessage = ""; From 7aeaa55413590c2f7dfb64dc8c53f30ee7ab449c Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 12:15:31 -0400 Subject: [PATCH 14/16] fix: hide the validation tab in live metrics for cross-validation --- DashAI/front/src/components/models/LiveMetricsChart.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DashAI/front/src/components/models/LiveMetricsChart.jsx b/DashAI/front/src/components/models/LiveMetricsChart.jsx index 47cef4af7..1b377a78d 100644 --- a/DashAI/front/src/components/models/LiveMetricsChart.jsx +++ b/DashAI/front/src/components/models/LiveMetricsChart.jsx @@ -362,7 +362,9 @@ export function LiveMetricsChart({ run, modelSessionDetail = null }) { {t("models:label.train")} - {isCV && ( + {/* Cross-validation folds every row into train and test, so those + runs have no validation partition to plot. */} + {!isCV && ( {t("models:label.validation")} From b3dd92aca1546a7c46daefe6334750cd72d67bf2 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 14:45:48 -0400 Subject: [PATCH 15/16] fix: only pass a random state to scikit-learn when shuffling --- DashAI/back/splitters/fold_splitter.py | 24 ++++++ DashAI/back/splitters/group_k_fold.py | 4 +- DashAI/back/splitters/k_fold.py | 4 +- .../back/splitters/stratified_group_k_fold.py | 4 +- DashAI/back/splitters/stratified_k_fold.py | 4 +- .../back/splitters/test_random_state_guard.py | 83 +++++++++++++++++++ 6 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 tests/back/splitters/test_random_state_guard.py diff --git a/DashAI/back/splitters/fold_splitter.py b/DashAI/back/splitters/fold_splitter.py index f1fb011c3..515cc4328 100644 --- a/DashAI/back/splitters/fold_splitter.py +++ b/DashAI/back/splitters/fold_splitter.py @@ -13,6 +13,30 @@ from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset +def sklearn_random_state(shuffle: bool, random_state: int): + """Adapt a seed to scikit-learn's shuffle contract. + + The ``KFold`` family raises when ``random_state`` is set while ``shuffle`` + is False, so the seed is only forwarded when it can have an effect. Meant + for the splitters that expose ``shuffle`` as a parameter; the repeated + splitters always shuffle and take no such argument, so they pass their seed + through directly. + + Parameters + ---------- + shuffle : bool + Whether the splitter shuffles before splitting. + random_state : int + The configured seed. + + Returns + ------- + int or None + ``random_state`` when shuffling, ``None`` otherwise. + """ + return random_state if shuffle else None + + class FoldSplitter(BaseSplitter): """Shared base class for splitters that generate multiple cross-validation folds. diff --git a/DashAI/back/splitters/group_k_fold.py b/DashAI/back/splitters/group_k_fold.py index 97405dff5..746a266ef 100644 --- a/DashAI/back/splitters/group_k_fold.py +++ b/DashAI/back/splitters/group_k_fold.py @@ -15,7 +15,7 @@ ) from DashAI.back.core.utils import MultilingualString -from .fold_splitter import FoldSplitter +from .fold_splitter import FoldSplitter, sklearn_random_state if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -197,7 +197,7 @@ def split_indexes( gkf = GroupKFold( n_splits=self.n_splits, shuffle=self.shuffle, - random_state=self.random_state if self.shuffle else None, + random_state=sklearn_random_state(self.shuffle, self.random_state), ) folds = list(gkf.split(indexes, groups=dataset_df_groups)) except ValueError as e: diff --git a/DashAI/back/splitters/k_fold.py b/DashAI/back/splitters/k_fold.py index d44bc51fd..25571f0a3 100644 --- a/DashAI/back/splitters/k_fold.py +++ b/DashAI/back/splitters/k_fold.py @@ -13,7 +13,7 @@ ) from DashAI.back.core.utils import MultilingualString -from .fold_splitter import FoldSplitter +from .fold_splitter import FoldSplitter, sklearn_random_state if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -139,7 +139,7 @@ def split_indexes( kf = KFold( n_splits=self.n_splits, shuffle=self.shuffle, - random_state=self.random_state if self.shuffle else None, + random_state=sklearn_random_state(self.shuffle, self.random_state), ) folds = list(kf.split(indexes)) except ValueError as e: diff --git a/DashAI/back/splitters/stratified_group_k_fold.py b/DashAI/back/splitters/stratified_group_k_fold.py index e67a75a45..8fe4869d1 100644 --- a/DashAI/back/splitters/stratified_group_k_fold.py +++ b/DashAI/back/splitters/stratified_group_k_fold.py @@ -15,7 +15,7 @@ ) from DashAI.back.core.utils import MultilingualString -from .fold_splitter import FoldSplitter +from .fold_splitter import FoldSplitter, sklearn_random_state if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -199,7 +199,7 @@ def split_indexes( sgkf = StratifiedGroupKFold( n_splits=self.n_splits, shuffle=self.shuffle, - random_state=self.random_state, + random_state=sklearn_random_state(self.shuffle, self.random_state), ) folds = list(sgkf.split(indexes, y=y_labels, groups=dataset_df_groups)) except ValueError as e: diff --git a/DashAI/back/splitters/stratified_k_fold.py b/DashAI/back/splitters/stratified_k_fold.py index 6a6232354..95dd3e56e 100644 --- a/DashAI/back/splitters/stratified_k_fold.py +++ b/DashAI/back/splitters/stratified_k_fold.py @@ -13,7 +13,7 @@ ) from DashAI.back.core.utils import MultilingualString -from .fold_splitter import FoldSplitter +from .fold_splitter import FoldSplitter, sklearn_random_state if TYPE_CHECKING: from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset @@ -137,7 +137,7 @@ def split_indexes( kf = StratifiedKFold( n_splits=self.n_splits, shuffle=self.shuffle, - random_state=self.random_state, + random_state=sklearn_random_state(self.shuffle, self.random_state), ) folds = list(kf.split(indexes, y=y_labels)) except ValueError as e: diff --git a/tests/back/splitters/test_random_state_guard.py b/tests/back/splitters/test_random_state_guard.py new file mode 100644 index 000000000..e8ca04146 --- /dev/null +++ b/tests/back/splitters/test_random_state_guard.py @@ -0,0 +1,83 @@ +"""Every splitter must keep scikit-learn's shuffle and random_state contract. + +scikit-learn rejects a non-null ``random_state`` when ``shuffle`` is False, so a +splitter that forwards both unconditionally crashes for any configuration that +turns shuffling off. +""" + +import pandas as pd +import pytest + +from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset +from DashAI.back.splitters.group_k_fold import GroupKFoldSplitter +from DashAI.back.splitters.holdout import HoldoutSplitter +from DashAI.back.splitters.k_fold import KFoldSplitter +from DashAI.back.splitters.stratified_group_k_fold import StratifiedGroupKFoldSplitter +from DashAI.back.splitters.stratified_k_fold import StratifiedKFoldSplitter + +FOLD_SPLITTERS = [KFoldSplitter, StratifiedKFoldSplitter] +GROUP_SPLITTERS = [GroupKFoldSplitter, StratifiedGroupKFoldSplitter] + + +@pytest.fixture +def xy(): + dataset = to_dashai_dataset( + pd.DataFrame( + { + "a": list(range(12)), + "group": [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "y": [0, 1] * 6, + } + ) + ) + return dataset.select_columns(["a", "group"]), dataset.select_columns(["y"]) + + +@pytest.mark.parametrize("splitter_cls", FOLD_SPLITTERS + GROUP_SPLITTERS) +def test_shuffle_off_with_a_random_state_still_splits(splitter_cls, xy): + x, y = xy + splitter = splitter_cls( + { + "n_splits": 3, + "shuffle": False, + "random_state": 42, + "group_column": "group", + } + ) + + folds = splitter.split_indexes(x, y) + + assert len(folds) == 3 + + +def test_holdout_shuffle_off_with_a_random_state_still_splits(xy): + x, y = xy + splitter = HoldoutSplitter( + { + "train": 0.6, + "test": 0.2, + "validation": 0.2, + "stratify": False, + "shuffle": False, + "random_state": 42, + } + ) + + train, test, val = splitter.split_indexes(x, y) + + assert len(train) + len(test) + len(val) == 12 + # Without shuffling the partitions keep the original row order. + assert train == sorted(train) + + +@pytest.mark.parametrize("splitter_cls", FOLD_SPLITTERS) +def test_shuffle_on_stays_reproducible(splitter_cls, xy): + x, y = xy + config = {"n_splits": 3, "shuffle": True, "random_state": 7} + + first = splitter_cls(config).split_indexes(x, y) + again = splitter_cls(config).split_indexes(x, y) + other = splitter_cls({**config, "random_state": 99}).split_indexes(x, y) + + assert [f[1].tolist() for f in first] == [f[1].tolist() for f in again] + assert [f[1].tolist() for f in first] != [f[1].tolist() for f in other] From 0081283b0319d4efe25704aa4444673a669a6ada Mon Sep 17 00:00:00 2001 From: Irozuku Date: Thu, 20 Aug 2026 17:13:19 -0400 Subject: [PATCH 16/16] docs: note that the random state is ignored without shuffling --- DashAI/back/splitters/group_k_fold.py | 41 ++++++++++++++----- DashAI/back/splitters/holdout.py | 38 ++++++++++++----- DashAI/back/splitters/k_fold.py | 41 ++++++++++++++----- .../back/splitters/stratified_group_k_fold.py | 41 ++++++++++++++----- DashAI/back/splitters/stratified_k_fold.py | 41 ++++++++++++++----- 5 files changed, 147 insertions(+), 55 deletions(-) diff --git a/DashAI/back/splitters/group_k_fold.py b/DashAI/back/splitters/group_k_fold.py index 746a266ef..0a28b2c87 100644 --- a/DashAI/back/splitters/group_k_fold.py +++ b/DashAI/back/splitters/group_k_fold.py @@ -78,11 +78,26 @@ class GroupKFoldSplitterSchema(BaseSchema): bool_field(), placeholder=False, description=MultilingualString( - en="Whether to shuffle the groups before assigning them to folds.", - es="Si se deben mezclar los grupos antes de asignarlos a las particiones.", - pt="Se os grupos devem ser embaralhados antes de atribuí-los às partições.", - de="Ob die Gruppen vor der Zuweisung zu Folds gemischt werden sollen.", - zh="分配到各折之前是否打乱分组。", + en=( + "Whether to shuffle the groups before assigning them to folds. When " + "shuffling is disabled, the random state has no effect." + ), + es=( + "Si se deben mezclar los grupos antes de asignarlos a las " + "particiones. Cuando la mezcla está desactivada, el estado aleatorio " + "no tiene efecto." + ), + pt=( + "Se os grupos devem ser embaralhados antes de atribuí-los às " + "partições. Quando o embaralhamento está desativado, o estado " + "aleatório não tem efeito." + ), + de=( + "Ob die Gruppen vor der Zuweisung zu Folds gemischt werden sollen. " + "Wenn das Mischen deaktiviert ist, hat der Zufallszustand keine " + "Wirkung." + ), + zh="分配到各折之前是否打乱分组。关闭打乱时,随机状态不起作用。", ), alias=MultilingualString( en="Shuffle", es="Mezclar", pt="Embaralhar", de="Mischen", zh="打乱" @@ -92,20 +107,24 @@ class GroupKFoldSplitterSchema(BaseSchema): int_field(ge=0), placeholder=42, description=MultilingualString( - en="Seed used to make the split reproducible when shuffle is enabled.", + en=( + "Seed used to make the split reproducible when shuffle is enabled. It " + "is ignored when shuffling is disabled." + ), es=( - "Semilla utilizada para que la división sea reproducible cuando " - "se activa la mezcla." + "Semilla utilizada para que la división sea reproducible cuando se " + "activa la mezcla. Se ignora cuando la mezcla está desactivada." ), pt=( "Semente usada para tornar a divisão reproduzível quando o " - "embaralhamento está ativado." + "embaralhamento está ativado. É ignorada quando o embaralhamento está " + "desativado." ), de=( "Seed, um die Aufteilung reproduzierbar zu machen, wenn Mischen " - "aktiviert ist." + "aktiviert ist. Wird ignoriert, wenn das Mischen deaktiviert ist." ), - zh="启用打乱时,用于使划分可复现的随机种子。", + zh="启用打乱时,用于使划分可复现的随机种子。关闭打乱时将被忽略。", ), alias=MultilingualString( en="Random state", diff --git a/DashAI/back/splitters/holdout.py b/DashAI/back/splitters/holdout.py index 10ea84974..71fbb882a 100644 --- a/DashAI/back/splitters/holdout.py +++ b/DashAI/back/splitters/holdout.py @@ -91,11 +91,23 @@ class HoldoutSplitterSchema(BaseSchema): bool_field(), placeholder=True, description=MultilingualString( - en="Whether to shuffle the data before splitting it.", - es="Si se deben mezclar los datos antes de dividirlos.", - pt="Se os dados devem ser embaralhados antes de dividi-los.", - de="Ob die Daten vor der Aufteilung gemischt werden sollen.", - zh="划分前是否打乱数据。", + en=( + "Whether to shuffle the data before splitting it. When shuffling is " + "disabled, the random state has no effect." + ), + es=( + "Si se deben mezclar los datos antes de dividirlos. Cuando la mezcla " + "está desactivada, el estado aleatorio no tiene efecto." + ), + pt=( + "Se os dados devem ser embaralhados antes de dividi-los. Quando o " + "embaralhamento está desativado, o estado aleatório não tem efeito." + ), + de=( + "Ob die Daten vor der Aufteilung gemischt werden sollen. Wenn das " + "Mischen deaktiviert ist, hat der Zufallszustand keine Wirkung." + ), + zh="划分前是否打乱数据。关闭打乱时,随机状态不起作用。", ), alias=MultilingualString( en="Shuffle", es="Mezclar", pt="Embaralhar", de="Mischen", zh="打乱" @@ -105,20 +117,24 @@ class HoldoutSplitterSchema(BaseSchema): int_field(ge=0), placeholder=42, description=MultilingualString( - en="Seed used to make the split reproducible when shuffle is enabled.", + en=( + "Seed used to make the split reproducible when shuffle is enabled. It " + "is ignored when shuffling is disabled." + ), es=( - "Semilla utilizada para que la división sea reproducible cuando " - "se activa la mezcla." + "Semilla utilizada para que la división sea reproducible cuando se " + "activa la mezcla. Se ignora cuando la mezcla está desactivada." ), pt=( "Semente usada para tornar a divisão reproduzível quando o " - "embaralhamento está ativado." + "embaralhamento está ativado. É ignorada quando o embaralhamento está " + "desativado." ), de=( "Seed, um die Aufteilung reproduzierbar zu machen, wenn Mischen " - "aktiviert ist." + "aktiviert ist. Wird ignoriert, wenn das Mischen deaktiviert ist." ), - zh="启用打乱时,用于使划分可复现的随机种子。", + zh="启用打乱时,用于使划分可复现的随机种子。关闭打乱时将被忽略。", ), alias=MultilingualString( en="Random state", diff --git a/DashAI/back/splitters/k_fold.py b/DashAI/back/splitters/k_fold.py index 25571f0a3..40220fec7 100644 --- a/DashAI/back/splitters/k_fold.py +++ b/DashAI/back/splitters/k_fold.py @@ -42,11 +42,26 @@ class KFoldSplitterSchema(BaseSchema): bool_field(), placeholder=True, description=MultilingualString( - en="Whether to shuffle the data before splitting it into folds.", - es="Si se deben mezclar los datos antes de dividirlos en particiones.", - pt="Se os dados devem ser embaralhados antes de dividi-los em partições.", - de="Ob die Daten vor der Aufteilung in Folds gemischt werden sollen.", - zh="划分为折之前是否打乱数据。", + en=( + "Whether to shuffle the data before splitting it into folds. When " + "shuffling is disabled, the random state has no effect." + ), + es=( + "Si se deben mezclar los datos antes de dividirlos en particiones. " + "Cuando la mezcla está desactivada, el estado aleatorio no tiene " + "efecto." + ), + pt=( + "Se os dados devem ser embaralhados antes de dividi-los em partições. " + "Quando o embaralhamento está desativado, o estado aleatório não tem " + "efeito." + ), + de=( + "Ob die Daten vor der Aufteilung in Folds gemischt werden sollen. " + "Wenn das Mischen deaktiviert ist, hat der Zufallszustand keine " + "Wirkung." + ), + zh="划分为折之前是否打乱数据。关闭打乱时,随机状态不起作用。", ), alias=MultilingualString( en="Shuffle", es="Mezclar", pt="Embaralhar", de="Mischen", zh="打乱" @@ -56,20 +71,24 @@ class KFoldSplitterSchema(BaseSchema): int_field(ge=0), placeholder=42, description=MultilingualString( - en="Seed used to make the split reproducible when shuffle is enabled.", + en=( + "Seed used to make the split reproducible when shuffle is enabled. It " + "is ignored when shuffling is disabled." + ), es=( - "Semilla utilizada para que la división sea reproducible cuando " - "se activa la mezcla." + "Semilla utilizada para que la división sea reproducible cuando se " + "activa la mezcla. Se ignora cuando la mezcla está desactivada." ), pt=( "Semente usada para tornar a divisão reproduzível quando o " - "embaralhamento está ativado." + "embaralhamento está ativado. É ignorada quando o embaralhamento está " + "desativado." ), de=( "Seed, um die Aufteilung reproduzierbar zu machen, wenn Mischen " - "aktiviert ist." + "aktiviert ist. Wird ignoriert, wenn das Mischen deaktiviert ist." ), - zh="启用打乱时,用于使划分可复现的随机种子。", + zh="启用打乱时,用于使划分可复现的随机种子。关闭打乱时将被忽略。", ), alias=MultilingualString( en="Random state", diff --git a/DashAI/back/splitters/stratified_group_k_fold.py b/DashAI/back/splitters/stratified_group_k_fold.py index 8fe4869d1..ea689af0a 100644 --- a/DashAI/back/splitters/stratified_group_k_fold.py +++ b/DashAI/back/splitters/stratified_group_k_fold.py @@ -78,11 +78,26 @@ class StratifiedGroupKFoldSplitterSchema(BaseSchema): bool_field(), placeholder=True, description=MultilingualString( - en="Whether to shuffle each group's samples before splitting.", - es="Si se deben mezclar las muestras de cada grupo antes de dividir.", - pt="Se as amostras de cada grupo devem ser embaralhadas antes de dividir.", - de="Ob die Proben jeder Gruppe vor der Aufteilung gemischt werden sollen.", - zh="划分前是否打乱每个分组内的样本。", + en=( + "Whether to shuffle each group's samples before splitting. When " + "shuffling is disabled, the random state has no effect." + ), + es=( + "Si se deben mezclar las muestras de cada grupo antes de dividir. " + "Cuando la mezcla está desactivada, el estado aleatorio no tiene " + "efecto." + ), + pt=( + "Se as amostras de cada grupo devem ser embaralhadas antes de " + "dividir. Quando o embaralhamento está desativado, o estado aleatório " + "não tem efeito." + ), + de=( + "Ob die Proben jeder Gruppe vor der Aufteilung gemischt werden " + "sollen. Wenn das Mischen deaktiviert ist, hat der Zufallszustand " + "keine Wirkung." + ), + zh="划分前是否打乱每个分组内的样本。关闭打乱时,随机状态不起作用。", ), alias=MultilingualString( en="Shuffle", es="Mezclar", pt="Embaralhar", de="Mischen", zh="打乱" @@ -92,20 +107,24 @@ class StratifiedGroupKFoldSplitterSchema(BaseSchema): int_field(ge=0), placeholder=42, description=MultilingualString( - en="Seed used to make the split reproducible when shuffle is enabled.", + en=( + "Seed used to make the split reproducible when shuffle is enabled. It " + "is ignored when shuffling is disabled." + ), es=( - "Semilla utilizada para que la división sea reproducible cuando " - "se activa la mezcla." + "Semilla utilizada para que la división sea reproducible cuando se " + "activa la mezcla. Se ignora cuando la mezcla está desactivada." ), pt=( "Semente usada para tornar a divisão reproduzível quando o " - "embaralhamento está ativado." + "embaralhamento está ativado. É ignorada quando o embaralhamento está " + "desativado." ), de=( "Seed, um die Aufteilung reproduzierbar zu machen, wenn Mischen " - "aktiviert ist." + "aktiviert ist. Wird ignoriert, wenn das Mischen deaktiviert ist." ), - zh="启用打乱时,用于使划分可复现的随机种子。", + zh="启用打乱时,用于使划分可复现的随机种子。关闭打乱时将被忽略。", ), alias=MultilingualString( en="Random state", diff --git a/DashAI/back/splitters/stratified_k_fold.py b/DashAI/back/splitters/stratified_k_fold.py index 95dd3e56e..079ea4eff 100644 --- a/DashAI/back/splitters/stratified_k_fold.py +++ b/DashAI/back/splitters/stratified_k_fold.py @@ -42,11 +42,26 @@ class StratifiedKFoldSplitterSchema(BaseSchema): bool_field(), placeholder=True, description=MultilingualString( - en="Whether to shuffle the data before splitting it into folds.", - es="Si se deben mezclar los datos antes de dividirlos en particiones.", - pt="Se os dados devem ser embaralhados antes de dividi-los em partições.", - de="Ob die Daten vor der Aufteilung in Folds gemischt werden sollen.", - zh="划分为折之前是否打乱数据。", + en=( + "Whether to shuffle the data before splitting it into folds. When " + "shuffling is disabled, the random state has no effect." + ), + es=( + "Si se deben mezclar los datos antes de dividirlos en particiones. " + "Cuando la mezcla está desactivada, el estado aleatorio no tiene " + "efecto." + ), + pt=( + "Se os dados devem ser embaralhados antes de dividi-los em partições. " + "Quando o embaralhamento está desativado, o estado aleatório não tem " + "efeito." + ), + de=( + "Ob die Daten vor der Aufteilung in Folds gemischt werden sollen. " + "Wenn das Mischen deaktiviert ist, hat der Zufallszustand keine " + "Wirkung." + ), + zh="划分为折之前是否打乱数据。关闭打乱时,随机状态不起作用。", ), alias=MultilingualString( en="Shuffle", es="Mezclar", pt="Embaralhar", de="Mischen", zh="打乱" @@ -56,20 +71,24 @@ class StratifiedKFoldSplitterSchema(BaseSchema): int_field(ge=0), placeholder=42, description=MultilingualString( - en="Seed used to make the split reproducible when shuffle is enabled.", + en=( + "Seed used to make the split reproducible when shuffle is enabled. It " + "is ignored when shuffling is disabled." + ), es=( - "Semilla utilizada para que la división sea reproducible cuando " - "se activa la mezcla." + "Semilla utilizada para que la división sea reproducible cuando se " + "activa la mezcla. Se ignora cuando la mezcla está desactivada." ), pt=( "Semente usada para tornar a divisão reproduzível quando o " - "embaralhamento está ativado." + "embaralhamento está ativado. É ignorada quando o embaralhamento está " + "desativado." ), de=( "Seed, um die Aufteilung reproduzierbar zu machen, wenn Mischen " - "aktiviert ist." + "aktiviert ist. Wird ignoriert, wenn das Mischen deaktiviert ist." ), - zh="启用打乱时,用于使划分可复现的随机种子。", + zh="启用打乱时,用于使划分可复现的随机种子。关闭打乱时将被忽略。", ), alias=MultilingualString( en="Random state",