fix(gooddata-eval): apply the forecasting review findings to what-if

Two of the three findings on #1798 are structural and apply here unchanged.

Tool calls were extracted from the current turn only. The agent may build the
scenario spec on one turn and execute it on the next -- the common path, since it
asks which measure to adjust first -- and reading a single turn dropped the
scenario the execution actually ran, failing a correct run for having no
adjustments. Extraction now reads every turn accumulated so far.

Unasserted content checks were published to Langfuse as BOOLEAN 1. They are True
internally so they cannot fail a run, but reporting that as a score claims the
evaluator verified something it never looked at. Only checks named in ev.asserted
are now scored.

The third finding (unchecked confidence/seasonality) was forecasting-specific.

1 test added, verified to fail against the previous version. 803 passed, lint and
format clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py

@@ -339,13 +339,16 @@ def _accumulate(result: ChatResult) -> None:
                     reasoning_steps.extend(partial.reasoning_steps or [])
                     response_id = partial.response_id or response_id
                     _accumulate(partial)
-                    create_args, execute_result = _extract_what_if_calls(partial.tool_call_events or [])
+                    create_args, execute_result = _extract_what_if_calls(all_tool_call_events)
                 turn_completed = False
                 break
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
-            create_args, execute_result = _extract_what_if_calls(chat_result.tool_call_events or [])
+            # Over every turn so far, not just this one: the agent may build the spec on
+            # one turn and execute it on the next, and reading a single turn would drop the
+            # scenario the execution actually ran.
+            create_args, execute_result = _extract_what_if_calls(all_tool_call_events)
             response_text = render_answer_text(chat_result)
             turn_completed = chat_result.stream_ended and bool(response_text)
             if execute_result is not None:
@@ -493,11 +496,22 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     "what_if_executed": ev.executed,
                     "what_if_success": ev.success,
                     "what_if_turn_completed": ev.turn_completed,
-                    "what_if_metric_correct": ev.metric_correct,
-                    "what_if_maql_correct": ev.maql_correct,
-                    "what_if_scenario_count_correct": ev.scenario_count_correct,
-                    "what_if_baseline_correct": ev.baseline_correct,
                 }
+                # Only the content checks the fixture actually pinned. An unasserted check
+                # is True internally so it cannot fail a run, but publishing that as a
+                # BOOLEAN 1 would claim the evaluator verified something it never looked at.
+                strict_checks.update(
+                    {
+                        key: value
+                        for name, key, value in (
+                            ("metric_id", "what_if_metric_correct", ev.metric_correct),
+                            ("scenario_maql", "what_if_maql_correct", ev.maql_correct),
+                            ("scenarios", "what_if_scenario_count_correct", ev.scenario_count_correct),
+                            ("include_baseline", "what_if_baseline_correct", ev.baseline_correct),
+                        )
+                        if name in ev.asserted
+                    }
+                )
                 with ctx.observe(pt, run_idx) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")

packages/gooddata-eval/tests/test_agentic_what_if.py

@@ -285,3 +285,20 @@ def test_a_wrong_adjustment_raises_naming_what_the_agent_actually_did():
     assert "* 3 *" in str(error)
     assert error.runs_passed == 0
     assert error.conversation_id == "conv-1"
+
+
+def test_a_spec_built_on_an_earlier_turn_is_still_the_one_scored():
+    """The agent may build the spec on one turn and execute it on the next -- reading only
+    the current turn's calls would drop the scenario the execution actually ran and fail a
+    correct run for having no adjustments."""
+    summary = _run(
+        [
+            _chat([_tc("create_what_if_scenario", _create_args())], text="Preparing the scenario."),
+            _chat([_tc("execute_what_if_scenario", {"scenario_ref": "wia_1"}, _OK_EXECUTE)]),
+        ]
+    )
+
+    assert summary.best.evaluation.executed is True
+    assert summary.best.evaluation.metric_correct is True
+    assert summary.best.evaluation.maql_correct is True
+    assert summary.pass_at_k is True