feat(gooddata-eval): evaluate the agentic dashboard-creation skill

Adds the agentic_dashboard_skill kind: it drives the conversation until the
agent produces a draft, then scores that draft.

The gate holds only what the agent decided -- the draft call succeeded, the
response carries the expected part, every expected chart is present, every
tab's date filter matches, and it authored at least the required number of
charts. Two observations sit beside the gate and never fail a run: whether a
set_skills call activated dashboard_builder, and whether the response's
references carried every widget id. The second one matters -- gen-ai rejects a
draft naming an unresolvable visualization before it can succeed, so a widget
missing from the references means reference building degraded on the way out,
which is a platform fault and must not be scored as the model's.

A chart the fixture gives an id is matched on that id alone, since the widget
title is title_override or fallback_title and the override is the model's own
choice; a differing title is reported as a note. A chart the fixture marks with
a null id has no identity to match on, so there the title is the match, taken
among the authored charts only -- matching it against every chart would let an
existing one of the same title satisfy it.

The simulated user is deterministic rather than an LLM call: when the agent
asks back instead of drafting, the only things it still needs are the charts
and the date range, and both come straight from the expectation. That reply is
byte-identical every turn, which is why the loop caps at one clarifying round
instead of inheriting metric_skill's seven.

The expectation is validated before the first API call: a fixture with no
visualizations would pass every chart check vacuously, and a date range the
simulated user cannot phrase would otherwise surface only on the branch where
the agent asks back, passing or crashing depending on what the model chose.

jira: QA-29347
risk: low
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: FrankHuynh

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -13,6 +13,7 @@
 from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
+from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
@@ -40,6 +41,7 @@ class _LfKw(TypedDict, total=False):
         "agentic_visualization",  # experimental: expected_output.expected_outputs (multi-candidate)
         "agentic_metric_skill",
         "agentic_alert_skill",
+        "agentic_dashboard_skill",
         "agentic_search",
         "agentic_general_question",
         "agentic_guardrail",
@@ -85,6 +87,10 @@ class _LfKw(TypedDict, total=False):
 # create_key_driver_analysis with no cleanup, and while the evaluator only ever reads that
 # call's ARGUMENTS -- never a created object id -- whether the platform persists anything is
 # unverified. Move it to the allowlist once someone confirms it does not.
+#
+# agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
+# any chart it authors in conversation state and writes neither until a user saves from the UI, so
+# it is a candidate for the allowlist once the dataset has runs behind it.
 WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
 
 
@@ -183,6 +189,18 @@ def _dispatch_agentic(
             agent_id=agent_id,
             **lf_kw,
         )
+    elif kind == "agentic_dashboard_skill":
+        return evaluate_agentic_dashboard_skill(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo if isinstance(eo, dict) else {},
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            **lf_kw,
+        )
     elif kind == "agentic_alert_skill":
         return evaluate_agentic_alert_skill(
             host=host,

packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py

@@ -16,6 +16,16 @@
     evaluate_agentic_conversation,
     run_agentic_conversation,
 )
+from gooddata_eval.core.agentic.dashboard_skill import (
+    AgenticDashboardSummary,
+    DashboardEvaluation,
+    DashboardRunResult,
+    DashboardSkillAssertionError,
+    build_simulated_reply,
+    evaluate_agentic_dashboard_skill,
+    evaluate_dashboard_draft,
+    run_agentic_dashboard_skill,
+)
 from gooddata_eval.core.agentic.general_question import (
     AgenticGeneralQuestionSummary,
     GeneralQuestionAssertionError,
@@ -62,6 +72,7 @@
 
 __all__ = [
     "AgenticAlertSummary",
+    "AgenticDashboardSummary",
     "AgenticGeneralQuestionSummary",
     "AgenticGuardrailSummary",
     "AgenticKdaSummary",
@@ -74,6 +85,9 @@
     "ConversationAssertionError",
     "ConversationFixture",
     "ConversationResult",
+    "DashboardEvaluation",
+    "DashboardRunResult",
+    "DashboardSkillAssertionError",
     "GeneralQuestionAssertionError",
     "GeneralQuestionResult",
     "GuardrailAssertionError",
@@ -91,6 +105,9 @@
     "VisualizationAssertionError",
     "evaluate_agentic_alert_skill",
     "evaluate_agentic_conversation",
+    "build_simulated_reply",
+    "evaluate_agentic_dashboard_skill",
+    "evaluate_dashboard_draft",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
     "evaluate_agentic_kda_skill",
@@ -99,6 +116,7 @@
     "evaluate_agentic_visualization",
     "run_agentic_alert_skill",
     "run_agentic_conversation",
+    "run_agentic_dashboard_skill",
     "run_agentic_general_question",
     "run_agentic_guardrail",
     "run_agentic_kda_skill",