Refuse a malformed spec instead of tracebacking (#200)

CLAUDE.md

@@ -854,15 +854,24 @@ edges and `definition_version`. When something about the spec's meaning looks
 wrong, the fix is in astra-tools, not here.
 
 **A spec ASTRA rejects never reaches a recipe.** `build` runs
-`validate_analysis_schema`, `validate_analysis_file` and
-`validate_universe_file` before resolving anything, and refuses with
+`validate_analysis_schema`, `validate_analysis_file`,
+`validate_universe_schema` and `validate_universe_file` — what
+`astra validate` runs — before resolving anything, and refuses with
 ASTRA's own errors. This is a contract requirement, not a courtesy:
 resolution answers what a *valid* spec means and does not re-check that
 it is one, so without the gate an invalid spec surfaces later as a
 missing decision or an unresolvable input — blaming the run for a fault
 in the file, far from the line at fault. It caught three of lc's own
 test fixtures the first time it ran.
 
+The gate runs in **two stages, in `astra validate`'s order**: the schema
+for the analysis and every universe first, and the semantic checks only
+once that passes. The order is load-bearing, not cosmetic — the semantic
+checks assume the structure holds, and `outputs: 5` or `decisions: [a]`
+crashed them out of `lc status` as a raw `TypeError` while both ran
+unconditionally. The price is that a spec with both kinds of fault is
+fixed in two passes rather than one.
+
 **One output is one file, and the spec names it.** An output is
 `results/<universe>/<output_id>.<format>` — never a directory a recipe
 fills — and the path in a rendered recipe *is* the path on disk: no
@@ -1092,18 +1101,17 @@ raised anywhere inside astra's validation or resolution was reported as a
 bad target — a rerun misdiagnosing itself, at the one place nobody is
 watching.
 
-**Known violation, unfixed: a malformed `astra.yaml` tracebacks out of
-`lc status` and `lc materialize --check`** as a raw
-`yaml.scanner.ScannerError`. `plan._validate` is the gate that turns a
-bad spec into ASTRA's own errors, but it only catches what *validation*
-reports — a spec that does not parse raises inside
-`validate_analysis_schema` → `astra.helpers.load_yaml` before there is
-anything to validate. (`lc materialize` usually masks it by refusing a
-dirty tree first, which is not a fix.) The shape of the fix is a
-`yaml.YAMLError` catch in `_validate` naming the file and the line, the
-same way the validator's own errors are rendered. Pre-existing and
-independent of the one-output-one-file change; written down here because
-it contradicts the invariant directly above it.
+**A file that is not a spec is a refusal, not a traceback.** Whatever
+can be wrong *in* a spec or universe file before there is anything to
+validate — bad YAML, not UTF-8, empty, a top-level list — reaches lc as
+one exception, astra's `SpecLoadError`, raised by the loader every
+validator goes through and naming the file (and the line and column for
+YAML). `_validate` catches exactly that and refuses with it; `lc run`'s
+`read_spec` refuses with it too, since the probe's readable inputs come
+from the file. Nothing broader is caught: an exception from inside astra
+on a file that *does* load is a bug in astra, and reporting it as "your
+spec does not validate" would be the misattribution `worker.main` was
+fixed for.
 
 **`git_sha` in a manifest is the commit the run *started* at**, not the
 commit the run went on to create. It is the code that produced the output.

src/lightcone/engine/plan.py

@@ -244,20 +244,36 @@ def _validate(spec_path: Path, universes: list[Path]) -> None:
 
     Raises:
         ProjectError: Listing every structural and semantic error found,
-            in ASTRA's own words.
+            in ASTRA's own words — or naming the one file that is not a
+            spec at all, which leaves nothing to validate.
     """
+    from astra.helpers import SpecLoadError
     from astra.validation import (
         validate_analysis_file,
         validate_analysis_schema,
         validate_universe_file,
+        validate_universe_schema,
     )
 
-    problems = [
-        *validate_analysis_schema(spec_path),
-        *(str(e) for e in validate_analysis_file(spec_path)),
-    ]
-    for path in universes:
-        problems += [f"{path.name}: {e}" for e in validate_universe_file(path, spec_path)]
+    try:
+        problems = [
+            *validate_analysis_schema(spec_path),
+            *(f"{path.name}: {e}" for path in universes for e in validate_universe_schema(path)),
+        ]
+        # The semantic checks assume the structure holds — a list where a
+        # mapping belongs is a crash in them, not a finding — so, as in
+        # `astra validate`, they run only once it does.
+        if not problems:
+            problems = [
+                *(str(e) for e in validate_analysis_file(spec_path)),
+                *(
+                    f"{path.name}: {e}"
+                    for path in universes
+                    for e in validate_universe_file(path, spec_path)
+                ),
+            ]
+    except SpecLoadError as exc:
+        problems = [str(exc)]
     if problems:
         listed = "\n".join(f"  {problem}" for problem in problems)
         raise ProjectError(

src/lightcone/engine/run.py

@@ -111,13 +111,21 @@ def read_spec(project: Path) -> dict[str, Any]:
     Returns:
         The spec with sub-analyses merged in; the top-level document alone
         if the tree will not resolve; an empty spec if there is none.
+
+    Raises:
+        ProjectError: If the spec exists but is not a spec at all — not
+            YAML, not UTF-8, or not a mapping — since nothing it declares
+            can be read.
     """
-    from astra.helpers import load_yaml, resolve_analysis_tree
+    from astra.helpers import SpecLoadError, load_yaml, resolve_analysis_tree
 
     spec_path = project / SPEC_FILENAME
     if not spec_path.exists():
         return {}
-    data: dict[str, Any] = load_yaml(spec_path)
+    try:
+        data = load_yaml(spec_path)
+    except SpecLoadError as exc:
+        raise ProjectError(f"{exc}\nThe probe reads the inputs this file declares.") from None
     try:
         return dict(resolve_analysis_tree(data, project))
     except Exception:

tests/test_cli.py

@@ -752,3 +752,24 @@ def test_status_has_exactly_one_flag(runner: CliRunner) -> None:
     assert "--json" in options
     for flag in ("--check", "--refresh", "--verbose", "--all"):
         assert flag not in options
+
+
+def test_status_on_a_spec_that_is_not_yaml_is_a_clean_error(
+    runner: CliRunner, project: Path
+) -> None:
+    """A read-only verb never tracebacks: the classic unquoted colon in a
+    recipe is a refusal naming the line."""
+    (project / ".python-version").write_text("3.13.0\n")
+    (project / "universes").mkdir()
+    (project / "universes" / "baseline.yaml").write_text("id: baseline\ndecisions: {}\n")
+    (project / "astra.yaml").write_text(
+        'version: "0.1"\nname: demo\noutputs:\n  - id: fit\n    type: metric\n'
+        "    format: json\n    recipe:\n      command: echo a: b > {output}\n"
+    )
+
+    result = runner.invoke(main, ["status"])
+
+    assert result.exit_code == 1
+    assert "does not validate" in result.output
+    assert "mapping values are not allowed here" in result.output
+    assert isinstance(result.exception, SystemExit)  # a refusal, not a traceback

tests/test_plan.py

@@ -203,6 +203,48 @@ def test_a_spec_astra_rejects_never_reaches_a_recipe(tmp_path: Path) -> None:
         _build(_project(tmp_path, spec))
 
 
+@pytest.mark.parametrize(
+    ("broken", "content", "fault"),
+    [
+        ("astra.yaml", b"name: a: b\n", "line 1, column 8"),
+        ("astra.yaml", b"", "found nothing"),
+        ("astra.yaml", "name: caf\u00e9\n".encode("latin-1"), "not UTF-8"),
+        ("universes/baseline.yaml", b"- baseline\n", "found a list"),
+    ],
+)
+def test_a_file_that_is_not_a_spec_is_refused_by_name(
+    tmp_path: Path, broken: str, content: bytes, fault: str
+) -> None:
+    """Nothing to validate, so ASTRA's loader names the file and the fault —
+    and the refusal is the gate's, not a parser traceback."""
+    root = _project(tmp_path)
+    (root / broken).write_bytes(content)
+    with pytest.raises(ProjectError, match="does not validate") as raised:
+        _build(root)
+    assert str(root / broken) in str(raised.value)
+    assert fault in str(raised.value)
+
+
+_EMPTY = 'version: "0.1"\nname: demo\ninputs: []\noutputs: []\ndecisions: {}\n'
+
+
+@pytest.mark.parametrize(
+    ("spec", "baseline", "fault"),
+    [
+        (_EMPTY.replace("outputs: []", "outputs: 5"), "id: baseline\ndecisions: {}\n", "outputs"),
+        (_EMPTY, "id: baseline\ndecisions: [mcmc]\n", "baseline.yaml: decisions"),
+    ],
+)
+def test_a_structural_error_is_refused_before_the_semantic_checks_run(
+    tmp_path: Path, spec: str, baseline: str, fault: str
+) -> None:
+    """The semantic checks assume the structure holds and crash on a list
+    where a mapping belongs, so they wait for the schema to pass — the
+    order `astra validate` runs them in, universe schema included."""
+    with pytest.raises(ProjectError, match=fault):
+        _build(_project(tmp_path, spec, baseline=baseline))
+
+
 # ---- ordering, closure, targets --------------------------------------------
 
 

tests/test_run.py

@@ -137,6 +137,14 @@ def test_a_missing_spec_reads_as_an_empty_one(project: Path) -> None:
 # ---- declared inputs ------------------------------------------------------
 
 
+def test_a_spec_that_is_not_a_spec_is_refused_by_name(project: Path) -> None:
+    """The probe's readable inputs are what the spec declares, so a file
+    that does not load is a refusal naming it, not a parser traceback."""
+    (project / "astra.yaml").write_text("name: a: b\n")
+    with pytest.raises(ProjectError, match="line 1, column 8"):
+        engine_run.read_spec(project)
+
+
 def test_declared_file_inputs_become_read_paths(project: Path) -> None:
     spec = engine_run.read_spec(project)
     assert engine_run.input_paths(project, spec) == [(project / "data" / "local.csv").resolve()]