diff --git a/.github/workflows/email_scorecard_refresh.yml b/.github/workflows/email_scorecard_refresh.yml index f2a49eb620..821b2dec01 100644 --- a/.github/workflows/email_scorecard_refresh.yml +++ b/.github/workflows/email_scorecard_refresh.yml @@ -188,7 +188,30 @@ env: NO_PROXY: "localhost,127.0.0.1" jobs: + plan-timeouts: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + benchmark: ${{ steps.budget.outputs.benchmark }} + job: ${{ steps.budget.outputs.job }} + steps: + - name: Calculate timeout budget + id: budget + shell: python + run: | + import os + limit = int(os.environ["LIMIT"]) + experiments = int(os.environ["EXPERIMENTS"]) + if limit < 1 or experiments < 1: + raise SystemExit("limit and experiments must be positive integers") + benchmark = (210 if limit >= 249 else 45) * experiments + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"benchmark={benchmark}\njob={benchmark + 230}\n") + refresh: + needs: plan-timeouts name: Re-run eval, refresh-or-reject scorecard # Require the WINDOWS stx runners: the bare 'stx' label also matches a # Linux box (xsj-aimlab-stxp-04), and this eval is Windows-only (PowerShell @@ -214,13 +237,14 @@ jobs: # same-version gate, cross-version gate, commit) @ 10 each -> 60 # job full: 210+35+45+25+35+60 = 410 -> timeout-minutes 440 (margin) # job subset: 45+35+45+25+35+60 = 245 -> timeout-minutes 275 (margin) + # Benchmark time scales with experiments; the 230-minute overhead is fixed. # These are CEILINGS, not expectations — expected wall-clock is ~2.5-3h # full, ~100min subset. The job timeout is only the backstop: every step # above the "short steps" bucket carries its OWN timeout-minutes so an # overrun fails at a NAMED step, never a silent job-level `cancelled` # (this is the root fix for #2094 — the old value, 90, wasn't even # derived from a measurement). - timeout-minutes: ${{ fromJSON(inputs.limit || '250') >= 249 && 440 || 275 }} + timeout-minutes: ${{ fromJSON(needs.plan-timeouts.outputs.job) }} steps: - name: Checkout (the pushed branch) uses: actions/checkout@v7 @@ -361,7 +385,7 @@ jobs: Write-Host "PROFILE: $profileName -- limit=$env:LIMIT experiments=$env:EXPERIMENTS ctx_size=$env:CTX_SIZE rebaseline=$env:REBASELINE | committed: limit=$COMMITTED_LIMIT n_runs=$COMMITTED_N ctx_size=$committedCtxDisplay aggregate=$committedAggDisplay | prev_tag=$PREV" - name: Run the email-triage benchmark (real eval) - timeout-minutes: ${{ fromJSON(inputs.limit || '250') >= 249 && 210 || 45 }} + timeout-minutes: ${{ fromJSON(needs.plan-timeouts.outputs.benchmark) }} env: # The agent's calendar-connector resolution blocks on the OS keyring in # a headless context — disable it so construction doesn't hang. diff --git a/hub/agents/email/python/packaging/assert_llm_coverage.py b/hub/agents/email/python/packaging/assert_llm_coverage.py index dcef0edc26..8f8a1288e4 100644 --- a/hub/agents/email/python/packaging/assert_llm_coverage.py +++ b/hub/agents/email/python/packaging/assert_llm_coverage.py @@ -23,6 +23,7 @@ import argparse import json +import math import os import sys from pathlib import Path @@ -98,6 +99,15 @@ def check(eval_dir: Path) -> int: _emit_summary("- **LLM coverage**: ❌ none — heuristic-only run, results void") return 1 + if ( + isinstance(classified, bool) + or not isinstance(classified, (int, float)) + or not math.isfinite(classified) + ): + print("ERROR: llm_classified_count must be a finite number.", file=sys.stderr) + _emit_summary("- **LLM coverage**: invalid classification count; results void") + return 1 + if float(classified) <= 0: print( f"ERROR: the eval classified 0 of {emails} messages with the LLM.\n" diff --git a/hub/agents/email/python/tests/test_assert_llm_coverage.py b/hub/agents/email/python/tests/test_assert_llm_coverage.py index 4ed331993b..de3b4c38b3 100644 --- a/hub/agents/email/python/tests/test_assert_llm_coverage.py +++ b/hub/agents/email/python/tests/test_assert_llm_coverage.py @@ -151,3 +151,14 @@ def test_verdict_written_to_job_summary( def test_main_accepts_dir_argument(self, tmp_path): assert assert_llm_coverage.main([str(_write_scorecard(tmp_path))]) == 0 + + +@pytest.mark.parametrize("value", ["garbage", [], {}, True, float("nan"), float("inf")]) +def test_invalid_classification_count_fails_without_traceback(tmp_path, capsys, value): + directory = _write_scorecard(tmp_path) + path = directory / "scorecard.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["performance"]["llm_classified_count"] = value + path.write_text(json.dumps(payload), encoding="utf-8") + assert assert_llm_coverage.check(directory) == 1 + assert "finite number" in capsys.readouterr().err diff --git a/tests/unit/eval/test_email_refresh_timeouts.py b/tests/unit/eval/test_email_refresh_timeouts.py new file mode 100644 index 0000000000..4d7d57efef --- /dev/null +++ b/tests/unit/eval/test_email_refresh_timeouts.py @@ -0,0 +1,53 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Execute the workflow's actual timeout calculation for repeat dispatches.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +@pytest.mark.parametrize( + "limit,experiments,benchmark_minutes,job", + [(250, 1, 210, 440), (250, 3, 630, 860), (25, 1, 45, 275), (25, 3, 135, 365)], +) +def test_repeat_budget_scales(limit, experiments, benchmark_minutes, job, tmp_path): + workflow = ( + Path(__file__).resolve().parents[3] + / ".github/workflows/email_scorecard_refresh.yml" + ) + jobs = yaml.safe_load(workflow.read_text(encoding="utf-8"))["jobs"] + script = jobs["plan-timeouts"]["steps"][0]["run"] + output = tmp_path / "output" + result = subprocess.run( + [sys.executable, "-c", script], + env=dict( + os.environ, + LIMIT=str(limit), + EXPERIMENTS=str(experiments), + GITHUB_OUTPUT=str(output), + ), + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert output.read_text() == f"benchmark={benchmark_minutes}\njob={job}\n" + assert jobs["refresh"]["needs"] == "plan-timeouts" + assert ( + jobs["refresh"]["timeout-minutes"] + == "${{ fromJSON(needs.plan-timeouts.outputs.job) }}" + ) + step = next( + step + for step in jobs["refresh"]["steps"] + if step["name"] == "Run the email-triage benchmark (real eval)" + ) + assert ( + step["timeout-minutes"] + == "${{ fromJSON(needs.plan-timeouts.outputs.benchmark) }}" + )