From 3e5d0e8af756aedba53d47e901d2519842da677b Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 14:54:51 +1100 Subject: [PATCH 1/8] feat(staged): classify failed prompts caused by the network Step 1 of the network pause plan: a pure classifier that answers whether a failed prompt was caused by the network, so a later step can park such a turn and resume it when the network returns. Nothing calls it yet. `classify_network_failure(provider, driver_error, provider_metadata)` lives in `session_runner/network_failure.rs` and returns the signal that matched, for logs and metrics. It reads the structured ACP error stored at `provider_metadata.acp.promptError` and checks, in order: 1. the run failed on the prompt, so a stale `promptError` is ignored; 2. the code is -32603; 3. `data.reason` is not `incoming_transport_closed`; 4. the agent's structured field allows a network cause: `errorKind` of `server_error` or `unknown` for Claude, `reason: turn_failed` for Goose; 5. a row of the fallback table matches for that agent. Only step 5 reads error prose, because no protocol or bridge version marks a failure as network-caused. The table sits in one commented block with the reason for each row: an allow-listed errno token in trailing parentheses, the exact `Request timed out` constant and the HTTP 200 gateway page for Claude, and the `Network error: ` prefix for Goose. TLS certificate codes and `ERR_PROXY_TUNNEL` are left out because they need the user. Codex is never classified. The driver's `Prompt failed:` prefix is now the exported constant `acp_client::PROMPT_FAILED_PREFIX`, used by `describe_prompt_error` and the recorded-turn path. The runner receives that text either bare or wrapped once as the `data` string of an "ACP protocol failed" error, so the prompt-failure check accepts the prefix at the start of the error or at the start of its outermost `data` string, and nowhere deeper. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- apps/staged/src-tauri/src/session_runner.rs | 3 + .../src/session_runner/network_failure.rs | 569 ++++++++++++++++++ crates/acp-client/src/driver.rs | 12 +- crates/acp-client/src/driver/recorded_turn.rs | 2 +- crates/acp-client/src/lib.rs | 1 + 5 files changed, 585 insertions(+), 2 deletions(-) create mode 100644 apps/staged/src-tauri/src/session_runner/network_failure.rs diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index e9260c765..13d2687e6 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -39,6 +39,9 @@ pub(crate) mod attachments; mod disconnect; mod late_stop; +// Nothing calls the classifier until network failures are parked as a pause. +#[cfg_attr(not(test), allow(dead_code))] +mod network_failure; pub(crate) mod reconcile; pub(crate) use disconnect::may_still_run_on_workstation; diff --git a/apps/staged/src-tauri/src/session_runner/network_failure.rs b/apps/staged/src-tauri/src/session_runner/network_failure.rs new file mode 100644 index 000000000..c92b3f48a --- /dev/null +++ b/apps/staged/src-tauri/src/session_runner/network_failure.rs @@ -0,0 +1,569 @@ +//! Was a failed prompt caused by the network? +//! +//! A turn that fails because the agent could not reach its model provider is +//! worth parking until the network returns; every other failure is not. This +//! module answers that one question from the structured ACP error stored at +//! `provider_metadata.acp.promptError` and the session's provider. It is a +//! pure function: it reads nothing but its arguments and decides nothing +//! about what happens next. +//! +//! Structured fields exclude every failure class that is not the network +//! (wrong code, a lost remote transport, a rate limit, a billing or sign-in +//! error). No version of the protocol or of the bridges marks a failure as +//! network-caused on the failed prompt itself, though, so the last split +//! rests on the error message. That fallback is confined to +//! [`matched_fallback_signal`] below. **No other file may match error prose +//! to decide whether a failure was the network.** + +use acp_client::PROMPT_FAILED_PREFIX; + +/// The JSON-RPC "internal error" code. Both bridges report a failed provider +/// request with it; `-32000` (authentication required) and every other code +/// are not network failures. +const INTERNAL_ERROR_CODE: i64 = -32603; + +/// `data.reason` on the error Staged's own driver writes when a remote +/// session's transport closes. That loss has its own recovery. +const INCOMING_TRANSPORT_CLOSED_REASON: &str = "incoming_transport_closed"; + +/// The `errorKind` values the Claude bridge can put on a network failure. +/// The CLI files connection errors, timeouts and real HTTP 5xx responses +/// alike under `server_error`, and a gateway's HTML page under `unknown`. +/// Every other kind, and a missing kind, is not a network failure. +const CLAUDE_NETWORK_ERROR_KINDS: &[&str] = &["server_error", "unknown"]; + +/// `data.reason` Goose puts on every failed turn, network or not. +const GOOSE_TURN_FAILED_REASON: &str = "turn_failed"; + +/// The signal that identified a failed prompt as a network failure, for logs +/// and metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NetworkFailure { + /// Claude: the CLI printed this errno token from its allow-list. + ClaudeErrno(&'static str), + /// Claude: the CLI's request timeout, which carries no token. + ClaudeRequestTimeout, + /// Claude: a proxy or gateway answered HTTP 200 with a page that is not + /// an API response. + ClaudeGatewayPage, + /// Goose: its `NetworkError` variant. + GooseNetworkError, +} + +impl NetworkFailure { + /// A stable name for the matched signal. + pub(crate) fn signal(&self) -> &'static str { + match self { + Self::ClaudeErrno(token) => token, + Self::ClaudeRequestTimeout => "request_timed_out", + Self::ClaudeGatewayPage => "gateway_page", + Self::GooseNetworkError => "goose_network_error", + } + } +} + +/// Classify a failed run as a network failure, or `None` for anything else. +/// +/// `driver_error` is the run's error as the driver reported it and +/// `provider_metadata` the session's stored metadata. The checks run in this +/// order, and only the last one reads error prose: +/// +/// 1. The run failed on the prompt. `promptError` can be left on the row by an +/// earlier event of the same turn, so a failure that is not a prompt +/// failure must not be judged by it. +/// 2. The error code is `-32603`. +/// 3. `data.reason` is not `incoming_transport_closed`. +/// 4. The agent's own structured field allows a network cause. +/// 5. A row of the fallback table matches for that agent. +/// +/// A rate limit never reaches step 5: its `errorKind` fails step 4. Callers +/// still run the rate-limit check first. +pub(crate) fn classify_network_failure( + provider: Option<&str>, + driver_error: &str, + provider_metadata: Option<&serde_json::Value>, +) -> Option { + let agent = NetworkAgent::from_provider(provider?)?; + if !failed_on_the_prompt(driver_error) { + return None; + } + let prompt_error = provider_metadata?.pointer("/acp/promptError")?; + if prompt_error.get("code").and_then(|code| code.as_i64()) != Some(INTERNAL_ERROR_CODE) { + return None; + } + let data = prompt_error.get("data"); + let data_str = |field: &str| { + data.and_then(|data| data.get(field)) + .and_then(|value| value.as_str()) + }; + if data_str("reason") == Some(INCOMING_TRANSPORT_CLOSED_REASON) { + return None; + } + let structured_gate_passes = match agent { + NetworkAgent::Claude => { + data_str("errorKind").is_some_and(|kind| CLAUDE_NETWORK_ERROR_KINDS.contains(&kind)) + } + NetworkAgent::Goose => data_str("reason") == Some(GOOSE_TURN_FAILED_REASON), + }; + if !structured_gate_passes { + return None; + } + let message = prompt_error.get("message").and_then(|m| m.as_str())?; + matched_fallback_signal(agent, message) +} + +/// The agents whose failed prompts can be classified. Codex is absent on +/// purpose: it retries a failed connection inside the turn without limit, and +/// its other failures end the turn as text, not as an error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NetworkAgent { + Claude, + Goose, +} + +impl NetworkAgent { + fn from_provider(provider: &str) -> Option { + match provider { + "claude" => Some(Self::Claude), + "goose" => Some(Self::Goose), + _ => None, + } + } +} + +/// Whether the driver reported a failed `session/prompt`. +/// +/// The driver's text starts with its own [`PROMPT_FAILED_PREFIX`]. The +/// connection that carried the prompt usually hands it on wrapped once, as +/// `ACP protocol failed: Error { ..., data: Some(String("Prompt failed: ...")) }`, +/// so the outermost `data` string is accepted too. Both forms are Staged's +/// own rendering, not agent prose. A deeper occurrence does not count: that +/// would be an agent quoting the words. +fn failed_on_the_prompt(driver_error: &str) -> bool { + if driver_error.starts_with(PROMPT_FAILED_PREFIX) { + return true; + } + // Outermost first, and only an unescaped (top-level) string counts. + const WRAPPED: &str = "data: Some(String(\""; + driver_error + .find(WRAPPED) + .is_some_and(|at| driver_error[at + WRAPPED.len()..].starts_with(PROMPT_FAILED_PREFIX)) +} + +// --------------------------------------------------------------------------- +// The fallback table: the only place that reads error prose. +// +// Each row applies only after the structured checks in +// `classify_network_failure` have passed, and only for its own agent. Each +// exists because the agent forwards no structured cause for that failure. +// When the Claude bridge starts copying the SDK's `api_error_status` into the +// prompt error's `data`, check that field ahead of this table and delete the +// Claude rows it covers. +// --------------------------------------------------------------------------- + +/// Claude row 1: an errno token in parentheses at the end of the message. +/// +/// The CLI takes the raw `code` from the error chain and prints it in +/// parentheses, so the token is a machine string even though it sits inside +/// prose; the bridge forwards no cause field. Only these tokens count. TLS +/// certificate codes and `ERR_PROXY_TUNNEL` are left out because waiting for +/// the network does not fix them: they need the user. +const CLAUDE_NETWORK_ERRNO_TOKENS: &[&str] = &[ + "ENOTFOUND", + "EAI_AGAIN", + "FailedToOpenSocket", + "ECONNREFUSED", + "ConnectionRefused", + "ECONNRESET", + "EPIPE", + "ECONNABORTED", + "ConnectionClosed", + "ERR_SOCKET_CLOSED", + "UND_ERR_SOCKET", + "ENETUNREACH", + "ENETDOWN", + "EHOSTUNREACH", + "EHOSTDOWN", + "ETIMEDOUT", +]; + +/// The prefix the Claude bridge puts on every failed provider request. +const CLAUDE_INTERNAL_ERROR_PREFIX: &str = "Internal error: "; + +/// Claude row 2: the whole message after [`CLAUDE_INTERNAL_ERROR_PREFIX`]. +/// A fixed English constant in the CLI, which emits no token for a timeout. +const CLAUDE_REQUEST_TIMED_OUT: &str = "Request timed out"; + +/// Claude row 3: both must appear. A gateway or captive-portal page answered +/// in place of the API; the bridge has no kind for it beyond `unknown`. +const CLAUDE_GATEWAY_PAGE_MARKERS: [&str; 2] = ["(HTTP 200)", "empty or malformed response"]; + +/// Goose row: the `Display` prefix of Goose's internal `NetworkError` +/// variant. `data.reason` is `turn_failed` for every Goose failure, so +/// nothing structured separates this from a 5xx or a rate limit. +const GOOSE_NETWORK_ERROR_PREFIX: &str = "Network error: "; + +fn matched_fallback_signal(agent: NetworkAgent, message: &str) -> Option { + let message = message.trim(); + match agent { + NetworkAgent::Claude => { + if let Some(token) = trailing_parenthesized_token(message) + .and_then(|token| CLAUDE_NETWORK_ERRNO_TOKENS.iter().find(|t| **t == token)) + { + return Some(NetworkFailure::ClaudeErrno(token)); + } + if message.strip_prefix(CLAUDE_INTERNAL_ERROR_PREFIX) == Some(CLAUDE_REQUEST_TIMED_OUT) + { + return Some(NetworkFailure::ClaudeRequestTimeout); + } + if CLAUDE_GATEWAY_PAGE_MARKERS + .iter() + .all(|marker| message.contains(marker)) + { + return Some(NetworkFailure::ClaudeGatewayPage); + } + None + } + NetworkAgent::Goose => message + .starts_with(GOOSE_NETWORK_ERROR_PREFIX) + .then_some(NetworkFailure::GooseNetworkError), + } +} + +/// The text inside a `(...)` that ends the message, if the message ends +/// with one. +fn trailing_parenthesized_token(message: &str) -> Option<&str> { + let inner = message.strip_suffix(')')?; + let open = inner.rfind('(')?; + Some(&inner[open + 1..]) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const DNS_MESSAGE: &str = "Internal error: API Error: Can't reach the API server — check your internet or DNS (ENOTFOUND)"; + const REFUSED_MESSAGE: &str = "Internal error: API Error: Connection refused — a firewall or proxy may be blocking it (ECONNREFUSED)"; + const TIMEOUT_MESSAGE: &str = "Internal error: Request timed out"; + const GATEWAY_MESSAGE: &str = "Internal error: API Error: API returned an empty or malformed response (HTTP 200) — check for a proxy or gateway intercepting the request. Response: content-type html, body is an HTML page, 62825 bytes, request-id absent, server cloudflare, intermediary headers cf-ray content-encoding transfer-encoding. This was the non-streaming retry of streaming request (no Anthropic request-id), which failed with: no_events, StreamNoEventsError; 0 stream events received."; + + fn metadata(prompt_error: serde_json::Value) -> serde_json::Value { + json!({ "acp": { "promptError": prompt_error } }) + } + + fn claude_error(kind: &str, message: &str) -> serde_json::Value { + metadata(json!({ "code": -32603, "message": message, "data": { "errorKind": kind } })) + } + + fn goose_error(message: &str) -> serde_json::Value { + metadata(json!({ "code": -32603, "message": message, "data": { "reason": "turn_failed" } })) + } + + /// The driver error as the runner receives it today: the prompt failure + /// wrapped once by the connection that carried it. + fn wrapped_driver_error(message: &str) -> String { + let inner = format!( + r#"Prompt failed: Error {{ code: -32603: Internal error, message: {message:?}, data: None }}"# + ); + format!( + r#"ACP protocol failed: Error {{ code: -32603: Internal error, message: "Internal error", data: Some(String({inner:?})) }}"# + ) + } + + fn classify(provider: &str, metadata: &serde_json::Value) -> Option { + let message = metadata + .pointer("/acp/promptError/message") + .and_then(|m| m.as_str()) + .unwrap_or_default(); + classify_network_failure( + Some(provider), + &wrapped_driver_error(message), + Some(metadata), + ) + } + + #[test] + fn the_four_stored_claude_network_payloads_classify_as_network() { + assert_eq!( + classify("claude", &claude_error("server_error", DNS_MESSAGE)), + Some(NetworkFailure::ClaudeErrno("ENOTFOUND")) + ); + assert_eq!( + classify("claude", &claude_error("server_error", REFUSED_MESSAGE)), + Some(NetworkFailure::ClaudeErrno("ECONNREFUSED")) + ); + // The stored timeout row carries `unknown`; `server_error` is what + // the CLI assigns today. + for kind in ["unknown", "server_error"] { + assert_eq!( + classify("claude", &claude_error(kind, TIMEOUT_MESSAGE)), + Some(NetworkFailure::ClaudeRequestTimeout), + "{kind}" + ); + } + assert_eq!( + classify("claude", &claude_error("unknown", GATEWAY_MESSAGE)), + Some(NetworkFailure::ClaudeGatewayPage) + ); + } + + #[test] + fn every_allow_listed_errno_token_matches_and_names_itself() { + for token in CLAUDE_NETWORK_ERRNO_TOKENS { + let message = format!("Internal error: API Error: Connection error ({token})"); + let failure = classify("claude", &claude_error("server_error", &message)) + .unwrap_or_else(|| panic!("{token} should classify")); + assert_eq!(failure, NetworkFailure::ClaudeErrno(token)); + assert_eq!(failure.signal(), *token); + } + } + + #[test] + fn the_bare_and_the_wrapped_driver_error_both_count_as_a_prompt_failure() { + let metadata = claude_error("server_error", DNS_MESSAGE); + let bare = format!( + r#"Prompt failed: Error {{ code: -32603: Internal error, message: {DNS_MESSAGE:?}, data: Some(Object {{"errorKind": String("server_error")}}) }}"# + ); + for driver_error in [bare, wrapped_driver_error(DNS_MESSAGE)] { + assert_eq!( + classify_network_failure(Some("claude"), &driver_error, Some(&metadata)), + Some(NetworkFailure::ClaudeErrno("ENOTFOUND")), + "{driver_error}" + ); + } + } + + #[test] + fn a_real_5xx_without_a_token_is_not_a_network_failure() { + for message in [ + "Internal error: API Error: 500 Internal server error", + "Internal error: API Error: Repeated 529 Overloaded errors", + // A token that is not at the end, or not in parentheses, is prose. + "Internal error: API Error: upstream said ECONNRESET while proxying", + "Internal error: API Error: (ECONNRESET) reported by upstream, HTTP 502", + // Not the whole message. + "Internal error: Request timed out while waiting for tool output", + "Internal error: API Error: Request timed out", + // One gateway marker without the other. + "Internal error: API Error: unexpected body (HTTP 200)", + ] { + assert_eq!( + classify("claude", &claude_error("server_error", message)), + None, + "{message}" + ); + } + } + + #[test] + fn a_tls_or_proxy_tunnel_code_needs_the_user_and_is_not_paused() { + for token in [ + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "SELF_SIGNED_CERT_IN_CHAIN", + "CERT_HAS_EXPIRED", + "ERR_PROXY_TUNNEL", + ] { + let message = format!("Internal error: API Error: Connection error ({token})"); + assert_eq!( + classify("claude", &claude_error("server_error", &message)), + None, + "{token}" + ); + } + } + + #[test] + fn every_other_claude_error_kind_is_excluded_even_with_a_network_message() { + for kind in [ + "authentication_failed", + "oauth_org_not_allowed", + "account_on_hold", + "verification_required", + "billing_error", + "rate_limit", + "overloaded", + "invalid_request", + "model_not_found", + "max_output_tokens", + "cloud_credential_error", + "incomplete_tool_call", + "no_result", + "worker_shutdown", + "transport_lost", + ] { + assert_eq!( + classify("claude", &claude_error(kind, DNS_MESSAGE)), + None, + "{kind}" + ); + } + } + + #[test] + fn a_missing_or_non_string_claude_error_kind_is_excluded() { + for data in [ + json!(null), + json!({}), + json!({ "errorKind": 7 }), + json!("server_error"), + ] { + let metadata = metadata(json!({ + "code": -32603, + "message": DNS_MESSAGE, + "data": data, + })); + assert_eq!(classify("claude", &metadata), None, "{data}"); + } + let no_data = metadata(json!({ "code": -32603, "message": TIMEOUT_MESSAGE })); + assert_eq!(classify("claude", &no_data), None); + } + + #[test] + fn a_rate_limit_is_never_a_network_failure() { + let metadata = metadata(json!({ + "code": -32603, + "message": "Internal error: API Error: Rate limited (ETIMEDOUT)", + "data": { "errorKind": "rate_limit", "rateLimit": { "resetsAt": 1_800_000_000 } }, + })); + assert_eq!(classify("claude", &metadata), None); + } + + #[test] + fn any_code_other_than_internal_error_is_excluded() { + let auth = metadata(json!({ "code": -32000, "message": "Authentication required" })); + assert_eq!(classify("claude", &auth), None); + for code in [ + json!(-32000), + json!(-32602), + json!(-32002), + json!("-32603"), + json!(null), + ] { + let metadata = metadata(json!({ + "code": code, + "message": DNS_MESSAGE, + "data": { "errorKind": "server_error" }, + })); + assert_eq!(classify("claude", &metadata), None, "{code}"); + } + } + + #[test] + fn a_lost_remote_transport_is_excluded() { + // As stored: written by Staged's own driver, with no error kind. + let stored = metadata(json!({ + "code": -32603, + "message": "Incoming transport closed", + "data": { "reason": "incoming_transport_closed", "method": "session/prompt" }, + })); + // Even if a kind and a network message rode along with it. + let with_kind = metadata(json!({ + "code": -32603, + "message": DNS_MESSAGE, + "data": { "reason": "incoming_transport_closed", "errorKind": "server_error" }, + })); + for provider in ["claude", "goose"] { + assert_eq!(classify(provider, &stored), None, "{provider}"); + assert_eq!(classify(provider, &with_kind), None, "{provider}"); + } + } + + #[test] + fn a_stale_prompt_error_is_ignored_when_the_run_did_not_fail_on_the_prompt() { + let metadata = claude_error("server_error", DNS_MESSAGE); + for driver_error in [ + "Timed out waiting for ACP protocol startup after 90s".to_string(), + "ACP protocol failed: server shut down unexpectedly".to_string(), + format!(r#"Prompt failed after interrupt: Error {{ message: {DNS_MESSAGE:?} }}"#), + format!(r#"Prompt failed after cancellation: Error {{ message: {DNS_MESSAGE:?} }}"#), + r#"ACP protocol failed: Error { code: -32603: Internal error, message: "Internal error", data: Some(String("ACP authentication is required to run the prompt. Sign this agent in, then retry.")) }"#.to_string(), + // The words quoted deeper down are an agent's, not the driver's. + r#"ACP protocol failed: Error { code: -32603: Internal error, message: "Internal error", data: Some(String("Failed to load session: Error { data: Some(String(\"Prompt failed: nope\")) }")) }"#.to_string(), + r#"Failed to load session: Error { message: "Prompt failed: (ENOTFOUND)", data: None }"#.to_string(), + String::new(), + ] { + assert_eq!( + classify_network_failure(Some("claude"), &driver_error, Some(&metadata)), + None, + "{driver_error}" + ); + } + } + + #[test] + fn a_prompt_failure_without_a_stored_prompt_error_is_not_classified() { + let driver_error = wrapped_driver_error(DNS_MESSAGE); + assert_eq!( + classify_network_failure(Some("claude"), &driver_error, None), + None + ); + assert_eq!( + classify_network_failure(Some("claude"), &driver_error, Some(&json!({ "acp": {} }))), + None + ); + } + + #[test] + fn goose_network_errors_classify_and_goose_server_errors_do_not() { + let network = goose_error( + "Network error: error sending request for url (https://api.example.com/v1/messages)", + ); + let failure = classify("goose", &network); + assert_eq!(failure, Some(NetworkFailure::GooseNetworkError)); + assert_eq!(failure.unwrap().signal(), "goose_network_error"); + + for message in [ + "Server error: 503 Service Unavailable", + "Rate limit exceeded: Network error: try later", + "Request failed: Network error: mentioned later (ENOTFOUND)", + ] { + assert_eq!(classify("goose", &goose_error(message)), None, "{message}"); + } + } + + #[test] + fn goose_needs_its_own_turn_failed_reason() { + for data in [ + json!({ "reason": "credits_exhausted" }), + json!({ "errorKind": "server_error" }), + json!(null), + ] { + let metadata = metadata(json!({ + "code": -32603, + "message": "Network error: connection reset", + "data": data, + })); + assert_eq!(classify("goose", &metadata), None, "{data}"); + } + } + + #[test] + fn each_fallback_row_applies_only_to_its_own_agent() { + // Goose's prefix under Claude's gate, and Claude's rows under Goose's. + assert_eq!( + classify( + "claude", + &claude_error("server_error", "Network error: connection reset") + ), + None + ); + for message in [DNS_MESSAGE, TIMEOUT_MESSAGE, GATEWAY_MESSAGE] { + assert_eq!(classify("goose", &goose_error(message)), None, "{message}"); + } + } + + #[test] + fn codex_and_unknown_providers_are_never_classified() { + let mut metadata = claude_error("server_error", DNS_MESSAGE); + metadata["acp"]["promptError"]["data"]["reason"] = json!("turn_failed"); + let driver_error = wrapped_driver_error(DNS_MESSAGE); + for provider in [Some("codex"), Some("amp"), Some(""), None] { + assert_eq!( + classify_network_failure(provider, &driver_error, Some(&metadata)), + None, + "{provider:?}" + ); + } + } +} diff --git a/crates/acp-client/src/driver.rs b/crates/acp-client/src/driver.rs index e3e8afa93..cee4447b0 100644 --- a/crates/acp-client/src/driver.rs +++ b/crates/acp-client/src/driver.rs @@ -7774,6 +7774,16 @@ async fn record_prompt_error( .await; } +/// How the driver's error text begins when the agent answered `session/prompt` +/// with an error other than `auth_required` (see [`describe_prompt_error`]). +/// +/// This is the driver's own wording, not the agent's, so a caller may test for +/// it to learn that a run failed on the prompt itself and not during setup, +/// after a cancel, or on a lost transport. The colon is part of the prefix: +/// "Prompt failed after interrupt: ..." and "Prompt failed after +/// cancellation: ..." are different failures and do not match. +pub const PROMPT_FAILED_PREFIX: &str = "Prompt failed:"; + /// The text a failed `session/prompt` surfaces to the user. /// /// amp, claude, and goose with a native provider accept `session/new` while @@ -7795,7 +7805,7 @@ fn describe_prompt_error( required.log_methods(&format!("required to {operation}")); return required.describe(operation); } - format!("Prompt failed: {error:?}") + format!("{PROMPT_FAILED_PREFIX} {error:?}") } /// What an agent said on a `-32000` beyond the schema's stock text, if anything. diff --git a/crates/acp-client/src/driver/recorded_turn.rs b/crates/acp-client/src/driver/recorded_turn.rs index b5969c9c5..e39db09e1 100644 --- a/crates/acp-client/src/driver/recorded_turn.rs +++ b/crates/acp-client/src/driver/recorded_turn.rs @@ -188,7 +188,7 @@ impl PromptAnswers { fn prompt_error_message(error: &serde_json::Value) -> String { match serde_json::from_value::(error.clone()) { Ok(error) => describe_prompt_error(error, &[]), - Err(_) => format!("Prompt failed: {error}"), + Err(_) => format!("{PROMPT_FAILED_PREFIX} {error}"), } } diff --git a/crates/acp-client/src/lib.rs b/crates/acp-client/src/lib.rs index ca4a69624..cbceaac66 100644 --- a/crates/acp-client/src/lib.rs +++ b/crates/acp-client/src/lib.rs @@ -43,6 +43,7 @@ pub use driver::{ RemoteProcessStatus, ReplayBoundary, SessionConnection, SessionLifetime, SessionSettleReason, SessionSettled, SteerError, SteerOutcome, Store, TransportLoss, TransportLossPhase, TurnControlHandle, TurnResult, BACKGROUND_CONTINUATION_ORIGIN, INTERRUPT_ACK_TIMEOUT, + PROMPT_FAILED_PREFIX, }; pub use simple::{run_acp_prompt, run_acp_prompt_with_interpreter_env_snapshot}; pub use types::{ From 3f7fe56aae1ad458647348161b6a0169e56a4568 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 15:00:18 +1100 Subject: [PATCH 2/8] feat(staged): park failed prompts caused by the network as a pause Step 2 of the network pause plan: wire the classifier in. A turn that fails because the agent could not reach its model provider is now parked and resumed by the scheduler, instead of ending as `error`/`crashed`. It runs under the existing per-session backoff, eight-attempt cap and 24 hour window; the shared gate that replaces those comes later. The decision for a run that ended in an agent error moves out of `finish_session_run` into `session_runner/failed_run.rs`. It tries, in order: a provider limit, a network failure, then an ordinary failure. The limit check stays first so a limit is never taken for the network. Pipeline sessions are never classified, so a push or pull that fails on the network still ends `error`/`crashed`. A network failure reuses the rate-limit pause: status `paused`, `CompletionReason::RateLimited`, `rate_limit_kind = "network"` and no reset time. No column, migration or type is added. The kind is the new constant `NETWORK_PAUSE_KIND`, and the comment on `CompletionReason::RateLimited` now covers both meanings. Because the pause is the same one, the pause write, the late-Stop handling on both sides of it, the scheduler's claim and the queue gate apply unchanged. An automatic resume of a network pause sends a prompt that tells the agent its turn was cut off and to check its work first. The choice lives in `auto_resume_prompt` and `RateLimitResumeTicket::prompt`; a message the user queued during the pause still replaces it. A network pause records a `network_pause` metric event with the matched signal, in place of a rate-limit hit. Tests cover: a network prompt error parks the row with kind `network` and schedules a retry; a pipeline session with the same error ends `error`/`crashed`; a Stop before or after the pause write ends the row `cancelled`; the auto-resume sends the network prompt; a queued message replaces it; and API-limit pauses keep their kind and prompt. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../src-tauri/src/rate_limit_metrics.rs | 20 ++ .../src-tauri/src/rate_limit_scheduler.rs | 112 ++++++- apps/staged/src-tauri/src/session_commands.rs | 25 +- apps/staged/src-tauri/src/session_runner.rs | 58 ++-- .../src/session_runner/failed_run.rs | 284 ++++++++++++++++++ .../src-tauri/src/session_runner/late_stop.rs | 77 ++++- apps/staged/src-tauri/src/store/models.rs | 4 +- 7 files changed, 535 insertions(+), 45 deletions(-) create mode 100644 apps/staged/src-tauri/src/session_runner/failed_run.rs diff --git a/apps/staged/src-tauri/src/rate_limit_metrics.rs b/apps/staged/src-tauri/src/rate_limit_metrics.rs index f54a8ffa2..343115847 100644 --- a/apps/staged/src-tauri/src/rate_limit_metrics.rs +++ b/apps/staged/src-tauri/src/rate_limit_metrics.rs @@ -88,6 +88,26 @@ pub(crate) fn record_rate_limit_hit( ); } +/// A session was paused because its prompt failed on the network. Counted +/// apart from rate-limit hits; `signal` is what identified the failure. +pub(crate) fn record_network_pause(session_id: &str, provider: Option<&str>, signal: &str) { + let provider = metric_label(provider); + let snapshot = with_state(|state| { + state.increment_counter("network_pauses_total"); + state.snapshot.clone() + }); + + log_metric( + "network_pause", + json!({ + "sessionId": session_id, + "provider": provider, + "signal": signal, + }), + snapshot, + ); +} + /// Observe the count of paused sessions the scheduler can still act on. /// /// Sessions that gave up or need a manual resume are excluded by the caller, diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index 98bd0474b..b4420c98b 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -1,4 +1,6 @@ -//! Jittered auto-recovery for sessions paused by provider API limits. +//! Jittered auto-recovery for sessions paused by provider API limits, and for +//! sessions whose prompt failed on the network ([`NETWORK_PAUSE_KIND`]), which +//! share the same pause. //! //! Retry bookkeeping is stored under `provider_metadata.staged` so it survives //! restarts without widening the stable session schema. The scheduler only @@ -18,6 +20,29 @@ use crate::actions::{ActionExecutor, ActionRegistry}; use crate::store::{MessageRole, Session, Store}; pub(crate) const AUTO_RETRY_RESUME_PROMPT: &str = "Continue where you left off."; +/// The resume prompt for a session paused by a network failure. The agent's +/// turn was cut off mid-request, so it is told to check its work first. +pub(crate) const NETWORK_RESUME_PROMPT: &str = "Your previous turn was cut off by a network error \ + before it finished. Check the state of your work, then continue where you left off."; + +/// The `rate_limit_kind` of a session paused because its prompt failed on the +/// network, not on a provider limit. +/// +/// Such a session shares the rate-limit pause (`paused` with +/// `CompletionReason::RateLimited`) so that Stop, the scheduler's claim and +/// the queue gate treat it the same way; this kind is the only thing that +/// tells the two apart. It carries no reset time. +pub(crate) const NETWORK_PAUSE_KIND: &str = "network"; + +/// The prompt an automatic resume sends when the user queued no message of +/// their own, chosen by what paused the session. +pub(crate) fn auto_resume_prompt(rate_limit_kind: Option<&str>) -> &'static str { + if rate_limit_kind == Some(NETWORK_PAUSE_KIND) { + NETWORK_RESUME_PROMPT + } else { + AUTO_RETRY_RESUME_PROMPT + } +} const TICK_INTERVAL_SECS: u64 = 15; const MAX_AUTO_RETRY_ATTEMPTS: u32 = 8; @@ -1518,6 +1543,91 @@ mod tests { ); } + /// A running session parked the way the runner parks a network failure. + fn pause_for_network(store: &Store, session_id: &str) { + assert!(pause_for_rate_limit( + store, + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + 10_000, + ) + .unwrap()); + } + + fn claimed_ticket( + store: &Arc, + session_id: &str, + ) -> Box { + match claim_for_auto_resume(store, session_id).unwrap() { + AutoResumeClaim::Claimed(ticket) => ticket, + _ => panic!("expected the idle branch to yield a claim"), + } + } + + #[test] + fn a_network_pause_is_scheduled_under_the_existing_backoff() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + pause_for_network(&store, &session.id); + + let listed = store.get_rate_limited_paused_sessions().unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].rate_limit_kind.as_deref(), Some("network")); + let state = auto_recovery_from_metadata(listed[0].provider_metadata.as_ref()).unwrap(); + assert_eq!(state.attempt, 1); + let next_retry_at = state.next_retry_at.expect("a retry is scheduled"); + assert!((10_000 + MIN_RETRY_DELAY_MS..=10_000 + BASE_BACKOFF_MS).contains(&next_retry_at)); + assert!(!is_due(&state, 10_000)); + assert!(is_due(&state, next_retry_at)); + } + + #[test] + fn auto_resume_of_a_network_pause_sends_the_network_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + pause_for_network(&store, &session.id); + + let ticket = claimed_ticket(&store, &session.id); + + assert_eq!(ticket.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(ticket.prompt(), NETWORK_RESUME_PROMPT); + assert!(NETWORK_RESUME_PROMPT + .starts_with("Your previous turn was cut off by a network error before it finished. ")); + } + + #[test] + fn a_queued_message_replaces_the_network_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + store + .add_queued_session_message(&session.id, "actually do this instead", &[], None) + .unwrap(); + pause_for_network(&store, &session.id); + + let ticket = claimed_ticket(&store, &session.id); + + assert_eq!(ticket.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(ticket.prompt(), "actually do this instead"); + } + + #[test] + fn auto_resume_of_an_api_limit_pause_keeps_the_plain_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, paused) = branch_with_paused_session(&store); + + assert_eq!( + claimed_ticket(&store, &paused.id).prompt(), + AUTO_RETRY_RESUME_PROMPT + ); + assert_eq!( + auto_resume_prompt(Some("five_hour")), + AUTO_RETRY_RESUME_PROMPT + ); + assert_eq!(auto_resume_prompt(None), AUTO_RETRY_RESUME_PROMPT); + } + #[test] fn preserves_only_staged_auto_recovery_metadata() { let metadata = json!({ diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index cd598781b..da9faf8fd 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -1788,6 +1788,20 @@ pub(crate) struct RateLimitResumeTicket { pub rate_limit_kind: Option, } +impl RateLimitResumeTicket { + /// What the resume sends: the user's queued message when there is one, + /// otherwise the synthetic prompt for what paused the session. + pub(crate) fn prompt(&self) -> String { + match &self.queued_message { + Some(message) => message.content.clone(), + None => { + crate::rate_limit_scheduler::auto_resume_prompt(self.rate_limit_kind.as_deref()) + .to_string() + } + } + } +} + pub(crate) async fn auto_resume_rate_limited_session( store: Arc, registry: Arc, @@ -1797,19 +1811,14 @@ pub(crate) async fn auto_resume_rate_limited_session( ticket: RateLimitResumeTicket, ) -> Result<(), String> { // The post-success drain cascade delivers any remaining queued messages. - let (prompt, image_ids, branch_id, queued_message_id) = match &ticket.queued_message { + let prompt = ticket.prompt(); + let (image_ids, branch_id, queued_message_id) = match &ticket.queued_message { Some(message) => ( - message.content.clone(), Some(message.image_ids.clone()), message.branch_id.clone(), Some(message.id.clone()), ), - None => ( - crate::rate_limit_scheduler::AUTO_RETRY_RESUME_PROMPT.to_string(), - None, - None, - None, - ), + None => (None, None, None), }; let result = resume_session_for_store( diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index 13d2687e6..4416af72d 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -38,9 +38,8 @@ pub(crate) mod attachments; mod disconnect; +mod failed_run; mod late_stop; -// Nothing calls the classifier until network failures are parked as a pause. -#[cfg_attr(not(test), allow(dead_code))] mod network_failure; pub(crate) mod reconcile; @@ -3310,45 +3309,44 @@ fn finish_session_run( .get_session(&session_id_for_status) .ok() .flatten(); - let provider_metadata = session_snapshot - .as_ref() - .and_then(|session| session.provider_metadata.as_ref()); let provider = session_snapshot .as_ref() .and_then(|session| session.provider.as_deref()) .or(config.provider.as_deref()); - // A rate limit pauses the session; anything else fails it, as a - // lost remote agent when that is what the error says. - let completion_reason = - match completion_reason_for_driver_error(message, provider_metadata) { - CompletionReason::Crashed => failed_run_completion_reason(message), - reason => reason, - }; - let rate_limit_recovery = - (completion_reason == CompletionReason::RateLimited).then(|| { - rate_limit_recovery_details( - message, - provider_metadata, - crate::store::now_timestamp(), - ) - }); - if let Some(recovery) = rate_limit_recovery.as_ref() { - crate::rate_limit_metrics::record_rate_limit_hit( + // A provider limit or a network failure pauses the session; + // anything else fails it (see `failed_run`). + let decision = failed_run::decide_failed_run( + session_snapshot.as_ref(), + config.provider.as_deref(), + message, + crate::store::now_timestamp(), + ); + match (decision.network_failure, decision.recovery.as_ref()) { + (Some(failure), _) => { + log::warn!( + "Session {session_id_for_status}: the prompt failed on the network \ + ({}); pausing it for an automatic resume", + failure.signal() + ); + crate::rate_limit_metrics::record_network_pause( + &session_id_for_status, + provider, + failure.signal(), + ); + } + (None, Some(recovery)) => crate::rate_limit_metrics::record_rate_limit_hit( &session_id_for_status, provider, recovery.kind.as_deref(), recovery.resets_at.is_some(), - ); + ), + (None, None) => {} } ( - if completion_reason == CompletionReason::RateLimited { - "paused" - } else { - "error" - }, + decision.status(), Some(innermost_error_sentence(message).unwrap_or_else(|| message.clone())), - completion_reason, - rate_limit_recovery, + decision.completion_reason, + decision.recovery, ) } }; diff --git a/apps/staged/src-tauri/src/session_runner/failed_run.rs b/apps/staged/src-tauri/src/session_runner/failed_run.rs new file mode 100644 index 000000000..f1f54b275 --- /dev/null +++ b/apps/staged/src-tauri/src/session_runner/failed_run.rs @@ -0,0 +1,284 @@ +//! How a run that ended in an agent error is recorded. +//! +//! Three outcomes, tried in this order: +//! +//! 1. **A provider limit** pauses the session until the limit resets. +//! 2. **A network failure** pauses it too, under the same pause with the +//! kind [`NETWORK_PAUSE_KIND`], so the scheduler resumes it. Pipeline +//! sessions are left out: a push or pull that fails on the network fails. +//! 3. **Anything else** fails the session, as a lost remote agent when that +//! is what the error says. +//! +//! The limit check runs first so a limit is never taken for the network. + +use super::network_failure::{classify_network_failure, NetworkFailure}; +use super::{ + completion_reason_for_driver_error, failed_run_completion_reason, rate_limit_recovery_details, + RateLimitRecovery, +}; +use crate::rate_limit_scheduler::NETWORK_PAUSE_KIND; +use crate::store::{CompletionReason, Session}; + +/// What [`decide_failed_run`] concluded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FailedRunDecision { + pub(super) completion_reason: CompletionReason, + /// Set exactly when the session is to be paused for an automatic resume. + pub(super) recovery: Option, + /// The signal that marked the failure as the network, when it paused the + /// session for that reason. + pub(super) network_failure: Option, +} + +impl FailedRunDecision { + /// The session status this decision is written as. + pub(super) fn status(&self) -> &'static str { + if self.recovery.is_some() { + "paused" + } else { + "error" + } + } +} + +/// Decide how a run that failed with `driver_error` ends. +/// +/// `session` is the row as it stands after the run, when it could be read; +/// `fallback_provider` is the run's configured provider, for a row that +/// records none. +pub(super) fn decide_failed_run( + session: Option<&Session>, + fallback_provider: Option<&str>, + driver_error: &str, + now_millis: i64, +) -> FailedRunDecision { + let provider_metadata = session.and_then(|session| session.provider_metadata.as_ref()); + if completion_reason_for_driver_error(driver_error, provider_metadata) + == CompletionReason::RateLimited + { + return FailedRunDecision { + completion_reason: CompletionReason::RateLimited, + recovery: Some(rate_limit_recovery_details( + driver_error, + provider_metadata, + now_millis, + )), + network_failure: None, + }; + } + + let provider = session + .and_then(|session| session.provider.as_deref()) + .or(fallback_provider); + let is_pipeline = session.is_some_and(|session| session.pipeline.is_some()); + let network_failure = if is_pipeline { + None + } else { + classify_network_failure(provider, driver_error, provider_metadata) + }; + if network_failure.is_some() { + return FailedRunDecision { + // The pause shared with provider limits; the kind tells them + // apart. No reset time: nothing says when the network returns. + completion_reason: CompletionReason::RateLimited, + recovery: Some(RateLimitRecovery { + resets_at: None, + kind: Some(NETWORK_PAUSE_KIND.to_string()), + }), + network_failure, + }; + } + + FailedRunDecision { + completion_reason: failed_run_completion_reason(driver_error), + recovery: None, + network_failure: None, + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serde_json::json; + + use super::*; + use crate::store::{PipelineExecution, PipelineKind, SessionStatus, Store}; + + const NOW_MS: i64 = 1_794_470_400_000; + const DNS_MESSAGE: &str = "Internal error: API Error: Can't reach the API server — check your internet or DNS (ENOTFOUND)"; + + /// The driver error as the runner receives it: the prompt failure wrapped + /// once by the connection that carried it. + fn driver_error(message: &str) -> String { + let inner = format!( + r#"Prompt failed: Error {{ code: -32603: Internal error, message: {message:?}, data: None }}"# + ); + format!( + r#"ACP protocol failed: Error {{ code: -32603: Internal error, message: "Internal error", data: Some(String({inner:?})) }}"# + ) + } + + fn prompt_error_metadata(kind: &str, message: &str) -> serde_json::Value { + json!({ "acp": { "promptError": { + "code": -32603, + "message": message, + "data": { "errorKind": kind }, + } } }) + } + + /// A Claude session whose prompt failed on a DNS lookup. + fn session_with_dns_failure() -> Session { + let mut session = Session::new_running("work", Path::new("/tmp")).with_provider("claude"); + session.provider_metadata = Some(prompt_error_metadata("server_error", DNS_MESSAGE)); + session + } + + #[test] + fn a_network_prompt_error_pauses_with_the_network_kind_and_no_reset() { + let session = session_with_dns_failure(); + + let decision = decide_failed_run(Some(&session), None, &driver_error(DNS_MESSAGE), NOW_MS); + + assert_eq!(decision.status(), "paused"); + assert_eq!(decision.completion_reason, CompletionReason::RateLimited); + assert_eq!( + decision.recovery, + Some(RateLimitRecovery { + resets_at: None, + kind: Some("network".to_string()), + }) + ); + assert_eq!( + decision.network_failure, + Some(NetworkFailure::ClaudeErrno("ENOTFOUND")) + ); + } + + /// The decision carried through the write `finish_session_run` makes with + /// it: the row is parked for the scheduler with kind `network`. + #[test] + fn a_network_prompt_error_parks_the_row_with_kind_network() { + let store = Store::in_memory().unwrap(); + let session = session_with_dns_failure().with_agent("agent-1"); + store.create_session(&session).unwrap(); + let row = store.get_session(&session.id).unwrap().unwrap(); + + let decision = decide_failed_run(Some(&row), None, &driver_error(DNS_MESSAGE), NOW_MS); + let recovery = decision.recovery.expect("a network failure pauses"); + assert!(crate::rate_limit_scheduler::pause_for_rate_limit( + &store, + &session.id, + Some("Can't reach the API server"), + recovery.resets_at.as_deref(), + recovery.kind.as_deref(), + NOW_MS, + ) + .unwrap()); + + let parked = store.get_session(&session.id).unwrap().unwrap(); + assert_eq!(parked.status, SessionStatus::Paused); + assert_eq!( + parked.completion_reason, + Some(CompletionReason::RateLimited) + ); + assert_eq!(parked.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(parked.rate_limit_resets_at, None); + let listed = store.get_rate_limited_paused_sessions().unwrap(); + assert_eq!(listed.len(), 1, "the scheduler lists the row"); + let bookkeeping = crate::rate_limit_scheduler::auto_recovery_from_metadata( + parked.provider_metadata.as_ref(), + ) + .expect("the pause writes retry bookkeeping"); + assert_eq!(bookkeeping.attempt, 1); + assert!(bookkeeping.next_retry_at.is_some()); + assert!(!bookkeeping.unsafe_to_auto_resume); + } + + #[test] + fn a_pipeline_session_with_the_same_error_fails_as_crashed() { + for kind in [PipelineKind::Push, PipelineKind::Pull] { + let mut session = session_with_dns_failure(); + session.pipeline = Some(PipelineExecution::from_steps(&[]).with_kind(kind)); + + let decision = + decide_failed_run(Some(&session), None, &driver_error(DNS_MESSAGE), NOW_MS); + + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + assert_eq!(decision.recovery, None); + assert_eq!(decision.network_failure, None); + } + } + + #[test] + fn the_configured_provider_stands_in_for_a_row_that_records_none() { + let mut session = session_with_dns_failure(); + session.provider = None; + let error = driver_error(DNS_MESSAGE); + + let with_fallback = decide_failed_run(Some(&session), Some("claude"), &error, NOW_MS); + assert_eq!(with_fallback.status(), "paused"); + + // Codex is never classified, and neither is an unknown agent. + for provider in [Some("codex"), None] { + let decision = decide_failed_run(Some(&session), provider, &error, NOW_MS); + assert_eq!(decision.status(), "error", "{provider:?}"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + } + } + + /// The limit check runs first and keeps its own kind and reset handling. + #[test] + fn a_rate_limit_is_still_a_rate_limit() { + let mut session = session_with_dns_failure(); + session.provider_metadata = Some(prompt_error_metadata( + "rate_limit", + "Internal error: rate limit exceeded", + )); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error("Internal error: rate limit exceeded"), + NOW_MS, + ); + + assert_eq!(decision.status(), "paused"); + assert_eq!(decision.completion_reason, CompletionReason::RateLimited); + assert_eq!( + decision.recovery.and_then(|recovery| recovery.kind), + Some("rate_limit".to_string()) + ); + assert_eq!(decision.network_failure, None); + } + + #[test] + fn other_failures_still_fail_the_session() { + // A real 5xx: the same structured kind, no network token. + let mut session = session_with_dns_failure(); + let message = "Internal error: API Error: 500 Internal server error"; + session.provider_metadata = Some(prompt_error_metadata("server_error", message)); + let decision = decide_failed_run(Some(&session), None, &driver_error(message), NOW_MS); + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + + // A row that could not be read has nothing to classify. + let decision = decide_failed_run(None, Some("claude"), &driver_error(DNS_MESSAGE), NOW_MS); + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + + // A lost remote agent keeps its own reason. + let decision = decide_failed_run( + None, + Some("claude"), + "ACP protocol failed: server shut down unexpectedly", + NOW_MS, + ); + assert_eq!(decision.status(), "error"); + assert_eq!( + decision.completion_reason, + failed_run_completion_reason("ACP protocol failed: server shut down unexpectedly") + ); + } +} diff --git a/apps/staged/src-tauri/src/session_runner/late_stop.rs b/apps/staged/src-tauri/src/session_runner/late_stop.rs index b98550e28..891fca9ce 100644 --- a/apps/staged/src-tauri/src/session_runner/late_stop.rs +++ b/apps/staged/src-tauri/src/session_runner/late_stop.rs @@ -9,9 +9,10 @@ //! - **Before the write** ([`terminal_decision_after_late_stop`]): the run //! reads the token once more and the Stop replaces whatever it had decided. //! - **After the write** ([`deregister_paused_run`]): for a terminal row there -//! is nothing left to stop. A row paused for rate-limit recovery is not -//! terminal — the scheduler restarts it — so the run reads the Stop out as -//! it deregisters and ends the row. +//! is nothing left to stop. A row paused for rate-limit recovery, or for +//! a network failure under the same pause, is not terminal — the scheduler +//! restarts it — so the run reads the Stop out as it deregisters and ends +//! the row. use super::{RateLimitRecovery, RegistrationEpoch, SessionRegistry}; use crate::store::{CompletionReason, SessionStatus, Store}; @@ -187,6 +188,16 @@ mod tests { } fn paused_run() -> PausedRun { + paused_run_for(Some(RESETS_AT), "five_hour") + } + + /// A run paused because its prompt failed on the network: the same pause + /// with the network kind and no reset time. + fn network_paused_run() -> PausedRun { + paused_run_for(None, crate::rate_limit_scheduler::NETWORK_PAUSE_KIND) + } + + fn paused_run_for(resets_at: Option<&str>, kind: &str) -> PausedRun { let store = Arc::new(Store::in_memory().unwrap()); let project = Project::new("test-owner/test-repo"); store.create_project(&project).unwrap(); @@ -209,8 +220,8 @@ mod tests { &store, &session.id, Some("rate limit"), - Some(RESETS_AT), - Some("five_hour"), + resets_at, + Some(kind), NOW_MS, ) .unwrap()); @@ -339,6 +350,62 @@ mod tests { } } + /// A network pause is the same pause, so a Stop on either side of its + /// write ends the row `cancelled` and the scheduler never restarts it. + #[test] + fn a_stop_during_a_network_pause_write_still_ends_cancelled() { + // Before the write: the Stop replaces the decision, recovery included. + let decided: TerminalDecision = ( + "paused", + Some("Can't reach the API server".to_string()), + CompletionReason::RateLimited, + Some(RateLimitRecovery { + resets_at: None, + kind: Some(crate::rate_limit_scheduler::NETWORK_PAUSE_KIND.to_string()), + }), + ); + assert_eq!( + terminal_decision_after_late_stop(decided, Some(CompletionReason::Interrupted)), + ("cancelled", None, CompletionReason::Interrupted, None) + ); + + // After the write: the deregister reads the Stop out and ends the row. + let run = network_paused_run(); + assert_eq!(run.row().status, SessionStatus::Paused); + assert_eq!(run.row().rate_limit_kind.as_deref(), Some("network")); + assert!(run.registry.cancel(&run.session_id)); + + assert!(run.deregister(&mock_app())); + + let row = run.row(); + assert_eq!(row.status, SessionStatus::Cancelled); + assert_eq!(row.completion_reason, Some(CompletionReason::Interrupted)); + assert!( + run.store + .get_rate_limited_paused_sessions() + .unwrap() + .is_empty(), + "the scheduler no longer lists the row" + ); + } + + /// With no Stop, a network-paused row stays for the scheduler. + #[test] + fn a_network_paused_run_no_stop_reached_stays_paused() { + let run = network_paused_run(); + + assert!(!run.deregister(&mock_app())); + + let row = run.row(); + assert_eq!(row.status, SessionStatus::Paused); + assert_eq!(row.completion_reason, Some(CompletionReason::RateLimited)); + assert_eq!(row.rate_limit_kind.as_deref(), Some("network")); + assert_eq!( + run.store.get_rate_limited_paused_sessions().unwrap().len(), + 1 + ); + } + /// The ordinary pause: no Stop reached the entry, so the row is left for /// the scheduler and the caller goes on to announce `paused`. #[test] diff --git a/apps/staged/src-tauri/src/store/models.rs b/apps/staged/src-tauri/src/store/models.rs index c7bc510a6..911787eae 100644 --- a/apps/staged/src-tauri/src/store/models.rs +++ b/apps/staged/src-tauri/src/store/models.rs @@ -574,7 +574,9 @@ pub enum CompletionReason { /// output ended, or bloxlet reports it stopped — e.g. bloxlet restarted /// under it. The error message names the cause. RemoteAgentLost, - /// Agent stopped because the provider reported a rate or usage limit. + /// Paused for an automatic resume. The provider reported a rate or usage + /// limit, or the prompt failed on the network: `rate_limit_kind` is + /// `"network"` for the latter. The name predates the network case. RateLimited, /// Staged closed while the session was still running. AppQuit, From d3ac2a27f7ec498433bcf84cec32b1f9c77f61ef Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 15:04:23 +1100 Subject: [PATCH 3/8] feat(staged): show network pause copy in the app Step 3 of the network pause plan, frontend only. A session parked because the agent could not reach its model provider is stored as a `rate_limited` pause with `rateLimitKind` of `network`, so until now the app described it as an API limit. It now says what happened. The timeline row subtitle from `failedArtifactSubtitle` reads "Paused: network unavailable. Resumes when the connection returns." for that kind. The chat alert from `sessionEndMessage` shows the same sentence, and adds "Trying again around 3:45 PM." when the scheduler has a next retry time. For a row the scheduler gave up on or will not resume by itself, the alert reads "Paused: network unavailable. Resume manually when ready.", leaving out the promise to resume because it would be false. The Resume now button is unchanged. API-limit pauses keep their copy. The kind is the constant `NETWORK_PAUSE_KIND` in `sessionFailureCopy.ts`, mirroring the backend constant of the same name. `failedArtifactSubtitle` now takes its optional session fields as one named object instead of four positional arguments. The project note row passed the reset time in the error-message position, so an API-limit pause on a project note never showed its reset time. Named fields fix that call and prevent the mix-up. The three `BranchTimeline.svelte` call sites and the project note row now pass `rateLimitKind`. Tests cover both kinds in the existing helper test files: the subtitle for each artifact, a network pause ignoring a provider reset time, the kind being read only on a rate-limit pause, the alert with and without a retry time, the manual-resume alert, and the project note row for a network pause and a dated API-limit pause. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../features/projects/projectNoteRow.test.ts | 27 +++++++ .../lib/features/projects/projectNoteRow.ts | 6 +- .../sessions/sessionModalHelpers.test.ts | 46 ++++++++++++ .../features/sessions/sessionModalHelpers.ts | 34 +++++++-- .../features/timeline/BranchTimeline.svelte | 35 +++++----- .../timeline/sessionFailureCopy.test.ts | 70 ++++++++++++++----- .../features/timeline/sessionFailureCopy.ts | 43 ++++++++++-- 7 files changed, 212 insertions(+), 49 deletions(-) diff --git a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts index 6e2c1f4d5..2e6add9c8 100644 --- a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts +++ b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts @@ -98,6 +98,33 @@ describe('projectNoteRow', () => { expect(failed(null).title).toBe('Session finished — no note created'); }); + it('titles a paused note by its kind of pause', () => { + const paused = (overrides: Partial) => + projectNoteRow( + note({ + sessionStatus: 'paused', + content: '', + title: '', + completionReason: 'rate_limited', + ...overrides, + }), + { deleting: false, nowMs: NOW } + ); + const resetsAt = '2026-06-24T15:45:00Z'; + const resetTime = new Date(resetsAt).toLocaleTimeString([], { + hour: 'numeric', + minute: '2-digit', + }); + expect(paused({ rateLimitKind: 'network' }).title).toBe( + 'Paused: network unavailable. Resumes when the connection returns.' + ); + expect(paused({}).title).toBe('Paused by API limit — no note yet'); + // The reset time dates the pause; it is not the session's error message. + expect(paused({ rateLimitResetsAt: resetsAt }).title).toBe( + `Paused by API limit — resets around ${resetTime}` + ); + }); + it('subtitles a deleting row "Deleting..." whatever its phase', () => { const live = { deleting: true, nowMs: NOW, liveHint: 'Reading' }; expect(projectNoteRow(note(), live).secondaryMeta).toBe('Deleting...'); diff --git a/apps/staged/src/lib/features/projects/projectNoteRow.ts b/apps/staged/src/lib/features/projects/projectNoteRow.ts index 084350e50..6ecd339d7 100644 --- a/apps/staged/src/lib/features/projects/projectNoteRow.ts +++ b/apps/staged/src/lib/features/projects/projectNoteRow.ts @@ -29,6 +29,7 @@ export type ProjectNoteRowSource = Pick< | 'updatedAt' | 'completedAt' | 'rateLimitResetsAt' + | 'rateLimitKind' >; export type ProjectNoteRowLive = { @@ -62,7 +63,10 @@ export function projectNoteRow( case 'failed': row = { type: 'failed-note', - title: failedArtifactSubtitle(note.completionReason, 'note', note.rateLimitResetsAt), + title: failedArtifactSubtitle(note.completionReason, 'note', { + rateLimitResetsAt: note.rateLimitResetsAt, + rateLimitKind: note.rateLimitKind, + }), secondaryMeta: undefined, }; break; diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts index 7907bcff2..12a33beeb 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts @@ -421,6 +421,52 @@ describe('newestUserMessageId', () => { 'This session paused because the provider reported an API limit. Resume manually when ready.' ); }); + + it('keeps the API limit copy for an API-limit kind', () => { + expect( + sessionEndMessage({ completionReason: 'rate_limited', rateLimitKind: 'five_hour' }) + ).toBe('This session paused because the provider reported an API limit.'); + }); + + it('explains a network pause', () => { + expect(sessionEndMessage({ completionReason: 'rate_limited', rateLimitKind: 'network' })).toBe( + 'Paused: network unavailable. Resumes when the connection returns.' + ); + }); + + it('says when a network pause tries again', () => { + const nextRetryAt = Date.UTC(2026, 5, 24, 15, 45); + const retryTime = new Date(nextRetryAt).toLocaleTimeString([], { + hour: 'numeric', + minute: '2-digit', + }); + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { + staged: { rateLimitAutoRecovery: { attempt: 2, maxAttempts: 8, nextRetryAt } }, + }, + }) + ).toBe( + `Paused: network unavailable. Resumes when the connection returns. Trying again around ${retryTime}.` + ); + }); + + it('asks for a manual resume of a network pause that will not resume by itself', () => { + for (const rateLimitAutoRecovery of [ + { attempt: 9, maxAttempts: 8, gaveUpAt: Date.UTC(2026, 5, 24, 15, 45) }, + { unsafeToAutoResume: true, reason: 'pending_tool_call' }, + ]) { + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { staged: { rateLimitAutoRecovery } }, + }) + ).toBe('Paused: network unavailable. Resume manually when ready.'); + } + }); }); describe('detached remote note sessions', () => { diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts index 7269551f5..8e12cfb6b 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts @@ -8,7 +8,12 @@ import { RECONCILE_WORKSTATION_GONE, isReconcileCancelNotice, } from '../../shared/disconnectedSession'; -import { formatRateLimitResetTime } from '../timeline/sessionFailureCopy'; +import { + NETWORK_PAUSED, + NETWORK_PAUSED_SUBTITLE, + formatRateLimitResetTime, + isNetworkPause, +} from '../timeline/sessionFailureCopy'; /** A text snippet attached to a new-session prompt (modal-local, never persisted). */ export interface TextSnippet { @@ -165,7 +170,9 @@ export function sessionEndNoticeKind( */ export function sessionEndMessage( current: Pick & - Partial> + Partial< + Pick + > ): string { switch (current.errorMessage) { case RECONCILE_TURN_UNDETERMINED: @@ -225,7 +232,7 @@ function detachedEndMessage(errorMessage: string | null | undefined): string { function endReasonMessage( current: Pick & - Partial> + Partial> ): string { switch (current.completionReason) { case 'crashed': @@ -254,12 +261,14 @@ function endReasonMessage( /** * Why a rate-limited session paused and what happens next: the scheduler's * bookkeeping in `providerMetadata.staged.rateLimitAutoRecovery` when it has - * any, else the provider's reset time. + * any, else the provider's reset time. A network pause is the same pause + * under another kind, and has its own copy. */ function rateLimitEndMessage( - current: Partial> + current: Partial> ): string { const recovery = rateLimitAutoRecovery(current.providerMetadata); + if (isNetworkPause(current.rateLimitKind)) return networkPauseEndMessage(recovery); if (recovery?.gaveUpAt) { return `This session paused because the provider reported an API limit. Auto-retry gave up after ${recovery.attempt ?? 'several'} attempts.`; } @@ -276,6 +285,21 @@ function rateLimitEndMessage( return 'This session paused because the provider reported an API limit.'; } +/** + * The notice for a network pause. A row the scheduler gave up on, or will not + * resume by itself, no longer resumes when the connection returns, so it says + * what to do instead of repeating that promise. + */ +function networkPauseEndMessage(recovery: RateLimitAutoRecovery | null): string { + if (recovery?.gaveUpAt || recovery?.unsafeToAutoResume) { + return `${NETWORK_PAUSED} Resume manually when ready.`; + } + if (recovery?.nextRetryAt) { + return `${NETWORK_PAUSED_SUBTITLE} Trying again around ${formatTimestamp(recovery.nextRetryAt)}.`; + } + return NETWORK_PAUSED_SUBTITLE; +} + /** * A failed session's error, as the chat pane shows it. Reconcile's terse * "workstation no longer exists" is reframed around what it means for the diff --git a/apps/staged/src/lib/features/timeline/BranchTimeline.svelte b/apps/staged/src/lib/features/timeline/BranchTimeline.svelte index f75a351c0..795798cc8 100644 --- a/apps/staged/src/lib/features/timeline/BranchTimeline.svelte +++ b/apps/staged/src/lib/features/timeline/BranchTimeline.svelte @@ -766,12 +766,11 @@ if (isFailed) { phase = 'failed'; type = 'failed-commit'; - secondaryMeta = failedArtifactSubtitle( - commit.completionReason, - 'commit', - commit.sessionErrorMessage, - commit.rateLimitResetsAt - ); + secondaryMeta = failedArtifactSubtitle(commit.completionReason, 'commit', { + errorMessage: commit.sessionErrorMessage, + rateLimitResetsAt: commit.rateLimitResetsAt, + rateLimitKind: commit.rateLimitKind, + }); } else if (isQueued) { phase = 'queued'; type = 'queued-commit'; @@ -840,13 +839,12 @@ if (phase === 'failed') { type = 'failed-note'; - secondaryMeta = failedArtifactSubtitle( - note.completionReason, - 'note', - note.sessionErrorMessage, - note.rateLimitResetsAt, - note.isPollingForNote - ); + secondaryMeta = failedArtifactSubtitle(note.completionReason, 'note', { + errorMessage: note.sessionErrorMessage, + rateLimitResetsAt: note.rateLimitResetsAt, + rateLimitKind: note.rateLimitKind, + isPollingForNote: note.isPollingForNote, + }); } else if (phase === 'queued') { type = 'queued-note'; secondaryMeta = queuedMeta; @@ -911,12 +909,11 @@ if (isFailed) { phase = 'failed'; type = 'failed-review'; - meta = failedArtifactSubtitle( - review.completionReason, - 'review', - review.sessionErrorMessage, - review.rateLimitResetsAt - ); + meta = failedArtifactSubtitle(review.completionReason, 'review', { + errorMessage: review.sessionErrorMessage, + rateLimitResetsAt: review.rateLimitResetsAt, + rateLimitKind: review.rateLimitKind, + }); } else if (isQueued) { phase = 'queued'; type = 'queued-review'; diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts index ca91345fb..1a8be5817 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts @@ -20,10 +20,10 @@ describe('failedArtifactSubtitle', () => { }); it('says "yet" for a detached remote session only while Staged is still working on its note', () => { - expect(failedArtifactSubtitle('detached', 'note', null, null, true)).toBe( + expect(failedArtifactSubtitle('detached', 'note', { isPollingForNote: true })).toBe( 'Lost the remote session — no note created yet' ); - expect(failedArtifactSubtitle('detached', 'note', null, null, false)).toBe( + expect(failedArtifactSubtitle('detached', 'note', { isPollingForNote: false })).toBe( 'Lost the remote session — no note created' ); expect(failedArtifactSubtitle('detached', 'note')).toBe( @@ -47,12 +47,12 @@ describe('failedArtifactSubtitle', () => { }); it('says a deleted workstation, not a crash, ended the session', () => { - expect(failedArtifactSubtitle('crashed', 'commit', RECONCILE_WORKSTATION_GONE)).toBe( - 'Workstation deleted — no commit created' - ); - expect(failedArtifactSubtitle('crashed', 'review', RECONCILE_WORKSTATION_GONE)).toBe( - 'Workstation deleted — no comments created' - ); + expect( + failedArtifactSubtitle('crashed', 'commit', { errorMessage: RECONCILE_WORKSTATION_GONE }) + ).toBe('Workstation deleted — no commit created'); + expect( + failedArtifactSubtitle('crashed', 'review', { errorMessage: RECONCILE_WORKSTATION_GONE }) + ).toBe('Workstation deleted — no comments created'); }); it('reads a turn Staged couldn’t follow as an interruption, not a Stop', () => { @@ -61,22 +61,24 @@ describe('failedArtifactSubtitle', () => { RECONCILE_NO_WORKSPACE, RECONCILE_PROMPT_NOT_RECEIVED, ]) { - expect(failedArtifactSubtitle('interrupted', 'note', message)).toBe( + expect(failedArtifactSubtitle('interrupted', 'note', { errorMessage: message })).toBe( 'Session interrupted — no note created' ); } }); it('keeps a Stop the workstation didn’t confirm a Stop', () => { - expect(failedArtifactSubtitle('interrupted', 'commit', RECONCILE_MAY_STILL_RUN)).toBe( - 'Session stopped — no commit created' - ); + expect( + failedArtifactSubtitle('interrupted', 'commit', { errorMessage: RECONCILE_MAY_STILL_RUN }) + ).toBe('Session stopped — no commit created'); }); it('falls back to the reason for any other message', () => { - expect(failedArtifactSubtitle('crashed', 'commit', 'Blox process exited with code 1')).toBe( - 'Session crashed — no commit created' - ); + expect( + failedArtifactSubtitle('crashed', 'commit', { + errorMessage: 'Blox process exited with code 1', + }) + ).toBe('Session crashed — no commit created'); }); it('explains API limit failures', () => { @@ -89,8 +91,42 @@ describe('failedArtifactSubtitle', () => { const resetTime = formatRateLimitResetTime('2026-06-24T15:45:00Z'); expect(resetTime).toBeTruthy(); - expect(failedArtifactSubtitle('rate_limited', 'commit', null, '2026-06-24T15:45:00Z')).toBe( - `Paused by API limit — resets around ${resetTime}` + expect( + failedArtifactSubtitle('rate_limited', 'commit', { + rateLimitResetsAt: '2026-06-24T15:45:00Z', + }) + ).toBe(`Paused by API limit — resets around ${resetTime}`); + }); + + it('keeps the API limit copy for an API-limit kind', () => { + expect(failedArtifactSubtitle('rate_limited', 'note', { rateLimitKind: 'five_hour' })).toBe( + 'Paused by API limit — no note yet' + ); + expect(failedArtifactSubtitle('rate_limited', 'note', { rateLimitKind: null })).toBe( + 'Paused by API limit — no note yet' + ); + }); + + it('says a network pause waits for the connection, whatever the artifact', () => { + for (const kind of ['commit', 'note', 'review'] as const) { + expect(failedArtifactSubtitle('rate_limited', kind, { rateLimitKind: 'network' })).toBe( + 'Paused: network unavailable. Resumes when the connection returns.' + ); + } + }); + + it('does not date a network pause by a provider reset time', () => { + expect( + failedArtifactSubtitle('rate_limited', 'commit', { + rateLimitKind: 'network', + rateLimitResetsAt: '2026-06-24T15:45:00Z', + }) + ).toBe('Paused: network unavailable. Resumes when the connection returns.'); + }); + + it('reads the network kind only on a rate-limit pause', () => { + expect(failedArtifactSubtitle('crashed', 'commit', { rateLimitKind: 'network' })).toBe( + 'Session crashed — no commit created' ); }); }); diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts index 57388235b..19a94fc65 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts @@ -24,19 +24,47 @@ const RECONCILE_INTERRUPTIONS: ReadonlySet = new Set([ RECONCILE_PROMPT_NOT_RECEIVED, ]); +/** + * The `rateLimitKind` of a `rate_limited` pause whose turn failed because the + * agent could not reach its model provider, not because of an API limit + * (`NETWORK_PAUSE_KIND` in `src-tauri/src/rate_limit_scheduler.rs`). + */ +export const NETWORK_PAUSE_KIND = 'network'; + +/** What a network pause says wherever it is named: the row subtitle and the chat alert. */ +export const NETWORK_PAUSED = 'Paused: network unavailable.'; +export const NETWORK_PAUSED_SUBTITLE = `${NETWORK_PAUSED} Resumes when the connection returns.`; + +/** Whether a `rate_limited` pause with this `rateLimitKind` is a network pause. */ +export function isNetworkPause(rateLimitKind: string | null | undefined): boolean { + return rateLimitKind === NETWORK_PAUSE_KIND; +} + +/** The session fields `failedArtifactSubtitle` reads beyond the completion reason. */ +export interface FailedArtifactDetails { + /** The session's error: reconcile's messages say more than its completion reason does. */ + errorMessage?: string | null; + /** Dates an API-limit pause's subtitle. */ + rateLimitResetsAt?: string | null; + /** Tells a network pause from an API-limit pause. */ + rateLimitKind?: string | null; + /** The note row's `NoteTimelineItem.isPollingForNote`. */ + isPollingForNote?: boolean; +} + /** * The subtitle of a timeline row whose session ended without producing its - * artifact. `errorMessage` is the session's: reconcile's messages say more - * than its completion reason does. `rateLimitResetsAt` dates a rate-limit - * pause's subtitle. `isPollingForNote` is the note row's - * `NoteTimelineItem.isPollingForNote`. + * artifact. */ export function failedArtifactSubtitle( completionReason: string | null | undefined, kind: ArtifactKind, - errorMessage?: string | null, - rateLimitResetsAt?: string | null, - isPollingForNote = false + { + errorMessage, + rateLimitResetsAt, + rateLimitKind, + isPollingForNote = false, + }: FailedArtifactDetails = {} ): string { const noun = artifactNoun[kind]; if (errorMessage === RECONCILE_WORKSTATION_GONE) { @@ -51,6 +79,7 @@ export function failedArtifactSubtitle( case 'remote_agent_lost': return `Workspace agent lost — no ${noun} created`; case 'rate_limited': + if (isNetworkPause(rateLimitKind)) return NETWORK_PAUSED_SUBTITLE; return rateLimitPausedSubtitle(rateLimitResetsAt) ?? `Paused by API limit — no ${noun} yet`; case 'app_quit': return `Session interrupted — no ${noun} created`; From fe444fe17a0543860a55de95d8f8042bfd0fac44 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 15:18:16 +1100 Subject: [PATCH 4/8] feat(staged): resume network pauses through one shared gate Step 4 of the network pause plan: network-paused sessions no longer back off one by one. They wait behind one gate held in memory by the scheduler, which resumes a single session as a probe and the rest once the probe gets through. API-limit pauses never consult it. The gate lives in `rate_limit_scheduler/network_gate.rs`, owned by `RateLimitScheduler` behind a mutex. It starts closed with a probe due at once, so a relaunch probes on its first tick. - Closed: a due tick resumes the oldest due, safe network row as the probe. Probing: no other network resume starts. Open: the rest resume at the existing two per tick. - A probe succeeds when its turn records agent output after the claim, or completes. The new store query `session_has_agent_output_since` backs that check and shares its row filter with `agent_answered_message`. - A probe fails when its session is parked again with kind `network`, or fails to start. The gate closes and the probe delay doubles from 15 seconds to a 10 minute cap. The re-parked row gets a new `updated_at`, so the next probe is a different session. - A probe that was stopped, deleted or ended on another error says nothing about the network, so another row may probe at once. - A failure while the gate is not open is not counted: `attempt` stays, `next_retry_at` is now, the cap and the 24 hour window are skipped, and the bookkeeping reason is `network_wait`. A failure while it is open is counted under the existing rules and closes the gate. `run_tick` settles the probe, then asks `admit_for_resume` for each row: the row's own retry state must be due, and a network row must be let through by the gate. The probe is recorded only after its claim wins, so a deferred or lost claim lets the next network row probe on that tick. `plan_recovery_state` and `build_recovery_state` take the kind and a `counted` flag. `pause_for_rate_limit` takes the scheduler, reports a network failure to the gate and plans the pause from its answer. It settles the probe first, so a probe that answered and then failed is counted. The runner passes the managed scheduler; with none, a network pause is not counted, as under a fresh gate. A failed resume start reports to the gate the same way. Tests drive the synchronous part of the tick with an injected `now`: one probe for eleven paused rows; a failed probe spends no attempt and rotates, with the delay doubling to its cap; agent output or completion opens the gate and the rest resume two per tick; output from before the claim does not; a failure while open advances `attempt` and closes the gate; a row past 24 hours still resumes; API-limit rows keep their schedule and never move the gate; a stopped probe hands over at once; a probe that fails to start closes the gate uncounted; and a fresh scheduler starts closed with an immediate probe. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../src-tauri/src/rate_limit_scheduler.rs | 253 ++++++++--- .../src/rate_limit_scheduler/network_gate.rs | 259 +++++++++++ .../network_gate/tests.rs | 408 ++++++++++++++++++ apps/staged/src-tauri/src/session_runner.rs | 6 + .../src/session_runner/failed_run.rs | 8 +- .../src-tauri/src/session_runner/late_stop.rs | 1 + apps/staged/src-tauri/src/store/messages.rs | 48 ++- 7 files changed, 925 insertions(+), 58 deletions(-) create mode 100644 apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs create mode 100644 apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index b4420c98b..263cf34b2 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -2,6 +2,10 @@ //! sessions whose prompt failed on the network ([`NETWORK_PAUSE_KIND`]), which //! share the same pause. //! +//! Network pauses do not back off one by one: they wait behind the shared gate +//! in [`network_gate`], which resumes one of them as a probe and the rest once +//! the probe gets through. +//! //! Retry bookkeeping is stored under `provider_metadata.staged` so it survives //! restarts without widening the stable session schema. The scheduler only //! resumes ACP sessions that can be replayed safely via `load_session`; sessions @@ -19,6 +23,10 @@ use tauri::AppHandle; use crate::actions::{ActionExecutor, ActionRegistry}; use crate::store::{MessageRole, Session, Store}; +mod network_gate; + +use network_gate::{NetworkGate, NetworkTurn}; + pub(crate) const AUTO_RETRY_RESUME_PROMPT: &str = "Continue where you left off."; /// The resume prompt for a session paused by a network failure. The agent's /// turn was cut off mid-request, so it is told to check its work first. @@ -44,6 +52,10 @@ pub(crate) fn auto_resume_prompt(rate_limit_kind: Option<&str>) -> &'static str } } +/// The bookkeeping `reason` of a network pause that waits for the gate +/// without spending a retry attempt. +const NETWORK_WAIT_REASON: &str = "network_wait"; + const TICK_INTERVAL_SECS: u64 = 15; const MAX_AUTO_RETRY_ATTEMPTS: u32 = 8; const GIVE_UP_AFTER_MS: i64 = 24 * 60 * 60 * 1_000; @@ -67,6 +79,8 @@ pub(crate) const MAX_PLAUSIBLE_RESET_LEAD_MS: i64 = 48 * 60 * 60 * 1_000; pub struct RateLimitScheduler { in_flight: Mutex>, tick_loop_spawned: AtomicBool, + /// The gate every network-paused session waits behind. + network_gate: Mutex, } impl RateLimitScheduler { @@ -161,6 +175,7 @@ async fn run_tick( let now = crate::store::now_timestamp(); let mut started = 0usize; let mut drained_branches: HashSet = HashSet::new(); + scheduler.settle_network_probe(&store, now); for session in sessions { if started >= MAX_RESUMES_PER_TICK { @@ -170,26 +185,11 @@ async fn run_tick( continue; } - let state = match ensure_auto_recovery_state(&store, &session, now) { - Ok(Some(state)) => state, - Ok(None) => { - scheduler.clear_in_flight(&session.id); - continue; - } - Err(e) => { - log::warn!( - "[rate_limit_recovery] failed to prepare session {}: {e}", - session.id - ); - scheduler.clear_in_flight(&session.id); - continue; - } - }; - - if !is_due(&state, now) { + let Some(admission) = admit_for_resume(&scheduler, &store, &session, now) else { scheduler.clear_in_flight(&session.id); continue; - } + }; + let state = admission.state; // The claim flips the row to `running` under our pid, and the runner // only registers it once the spawned task reaches `start_session` @@ -235,6 +235,11 @@ async fn run_tick( } }; + // Only now is the probe real: a deferred or lost claim above leaves + // the gate closed, so the next network row on this tick may probe. + if admission.probe { + scheduler.network_probe_started(&session.id, now); + } started += 1; crate::rate_limit_metrics::record_recovery_retry_attempt( &session.id, @@ -289,6 +294,7 @@ async fn run_tick( ); reschedule_after_start_failure( &store_for_task, + &scheduler_for_task, &session_id, crate::store::now_timestamp(), ); @@ -326,6 +332,52 @@ fn spawn_queue_drain(app_handle: &AppHandle, branch_id: &str) { ); } +/// A paused session the scheduler may claim on this tick. +struct ResumeAdmission { + state: RateLimitAutoRecovery, + /// The session resumes as the network gate's probe. + probe: bool, +} + +/// Whether `session` may be claimed on this tick: its own retry state is +/// due, and, for a network pause, the shared gate lets it through. +/// +/// API-limit pauses never consult the gate. +fn admit_for_resume( + scheduler: &RateLimitScheduler, + store: &Store, + session: &Session, + now: i64, +) -> Option { + let state = match ensure_auto_recovery_state(store, session, now) { + Ok(state) => state?, + Err(e) => { + log::warn!( + "[rate_limit_recovery] failed to prepare session {}: {e}", + session.id + ); + return None; + } + }; + if !is_due(&state, now) { + return None; + } + let probe = if is_network_pause(session.rate_limit_kind.as_deref()) { + match scheduler.network_turn(now) { + NetworkTurn::Hold => return None, + NetworkTurn::Probe => true, + NetworkTurn::Resume => false, + } + } else { + false + }; + Some(ResumeAdmission { state, probe }) +} + +fn is_network_pause(rate_limit_kind: Option<&str>) -> bool { + rate_limit_kind == Some(NETWORK_PAUSE_KIND) +} + fn is_due(state: &RateLimitAutoRecovery, now: i64) -> bool { if state.unsafe_to_auto_resume || state.gave_up_at.is_some() { return false; @@ -345,7 +397,9 @@ fn ensure_auto_recovery_state( return Ok(Some(state)); } - record_rate_limit_pause_for_session(store, session, now)?; + // No failure was reported for this row, so a network pause among these + // is not counted: it waits for the gate like the rest. + record_rate_limit_pause_for_session(store, session, false, now)?; Ok(store .get_session(&session.id) .map_err(|e| e.to_string())? @@ -366,11 +420,19 @@ fn ensure_auto_recovery_state( /// row's column (not yet written) and from the row as it stands, which is all /// the blocker scan needs: `agent_id` and the message history, not status. /// +/// A network pause (`kind` of [`NETWORK_PAUSE_KIND`]) is reported to the +/// scheduler's network gate, which answers whether it counts against the +/// session's retry budget: only a failure while the gate is open does. The +/// gate hears of the failure even if a Stop then wins the row, because the +/// network failed either way. With no scheduler the pause is not counted, as +/// under a fresh gate. +/// /// Returns whether this call won the `running → paused` transition. A lost /// race (a Stop cancelled the row first) writes nothing, including the /// fragment. pub(crate) fn pause_for_rate_limit( store: &Store, + scheduler: Option<&RateLimitScheduler>, session_id: &str, error_message: Option<&str>, resets_at: Option<&str>, @@ -380,7 +442,12 @@ pub(crate) fn pause_for_rate_limit( let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(false); }; - let state = plan_recovery_state(store, &session, resets_at, now)?; + let counted = if is_network_pause(kind) { + scheduler.is_some_and(|scheduler| scheduler.record_network_failure(store, session_id, now)) + } else { + true + }; + let state = plan_recovery_state(store, &session, kind, counted, resets_at, now)?; let transitioned = store .transition_running_to_rate_limited_with_provider_metadata( session_id, @@ -401,7 +468,7 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(()); }; - record_rate_limit_pause_for_session(store, &session, now) + record_rate_limit_pause_for_session(store, &session, true, now) } /// Write retry bookkeeping onto a row that is already paused. @@ -412,9 +479,17 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< fn record_rate_limit_pause_for_session( store: &Store, session: &Session, + counted: bool, now: i64, ) -> Result<(), String> { - let state = plan_recovery_state(store, session, session.rate_limit_resets_at.as_deref(), now)?; + let state = plan_recovery_state( + store, + session, + session.rate_limit_kind.as_deref(), + counted, + session.rate_limit_resets_at.as_deref(), + now, + )?; store .merge_session_provider_metadata(&session.id, &metadata_for_state(&state)) .map_err(|e| e.to_string())?; @@ -423,10 +498,12 @@ fn record_rate_limit_pause_for_session( } /// The next retry state for `session`, advancing any bookkeeping it already -/// carries. +/// carries. `kind` and `counted` are those of [`build_recovery_state`]. fn plan_recovery_state( store: &Store, session: &Session, + kind: Option<&str>, + counted: bool, resets_at: Option<&str>, now: i64, ) -> Result { @@ -435,6 +512,8 @@ fn plan_recovery_state( let jitter_ms = jitter_ms(jitter_bound_for(resets_at, now)); Ok(build_recovery_state( existing.as_ref(), + kind, + counted, resets_at, now, blocker.as_deref(), @@ -498,13 +577,24 @@ fn log_recovery_state(session: &Session, kind: Option<&str>, state: &RateLimitAu /// the 24h window in [`build_recovery_state`] still bound repeated failures, /// so give-up is reserved for starts that keep failing rather than issued on /// the first one. -fn reschedule_after_start_failure(store: &Store, session_id: &str, now: i64) { +/// +/// A network pause that fails to start is reported to the network gate like +/// a failed prompt, so a probe that could not start closes the gate again and +/// is counted only if the gate was open. +fn reschedule_after_start_failure( + store: &Store, + scheduler: &RateLimitScheduler, + session_id: &str, + now: i64, +) { match store.get_session(session_id) { // Only reschedule a session the failed attempt actually parked; a // session in any other state was claimed by something else (a manual // resume, a cancel, a delete) and is that path's to manage. Ok(Some(session)) if session.status.as_str() == "paused" => { - if let Err(e) = record_rate_limit_pause_for_session(store, &session, now) { + let counted = !is_network_pause(session.rate_limit_kind.as_deref()) + || scheduler.record_network_failure(store, session_id, now); + if let Err(e) = record_rate_limit_pause_for_session(store, &session, counted, now) { log::warn!( "[rate_limit_recovery] failed to reschedule session {session_id} after a failed start: {e}" ); @@ -530,8 +620,18 @@ fn is_actionable_for_auto_resume(session: &Session) -> bool { .unwrap_or(true) } +/// The retry state after one more pause. +/// +/// A network pause (`kind` of [`NETWORK_PAUSE_KIND`]) that is not `counted` +/// failed while the network gate was not open. It keeps its `attempt`, is due +/// at once so the gate alone decides when it resumes, and is exempt from the +/// attempt cap and the 24 hour window. Every other pause, a counted network +/// one included, follows the rules below; `counted` means nothing for an +/// API-limit pause. fn build_recovery_state( existing: Option<&RateLimitAutoRecovery>, + kind: Option<&str>, + counted: bool, resets_at: Option<&str>, now: i64, blocker: Option<&str>, @@ -552,6 +652,19 @@ fn build_recovery_state( }; } + if is_network_pause(kind) && !counted { + return RateLimitAutoRecovery { + attempt: existing.map(|s| s.attempt).unwrap_or(0), + max_attempts: MAX_AUTO_RETRY_ATTEMPTS, + next_retry_at: Some(now), + first_limited_at, + updated_at: now, + gave_up_at: None, + unsafe_to_auto_resume: false, + reason: Some(NETWORK_WAIT_REASON.to_string()), + }; + } + let attempt = existing.map(|s| s.attempt.saturating_add(1)).unwrap_or(1); if attempt > MAX_AUTO_RETRY_ATTEMPTS { return RateLimitAutoRecovery { @@ -943,7 +1056,7 @@ mod tests { let now = parse_rfc3339_millis("2026-06-24T15:00:00Z").unwrap(); let reset = "2026-06-24T15:45:00Z"; - let state = build_recovery_state(None, Some(reset), now, None, 12_000); + let state = build_recovery_state(None, None, true, Some(reset), now, None, 12_000); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("reset_at")); @@ -955,7 +1068,7 @@ mod tests { #[test] fn schedules_unknown_reset_with_capped_backoff_jitter() { - let state = build_recovery_state(None, None, 1_000_000, None, 4_000); + let state = build_recovery_state(None, None, true, None, 1_000_000, None, 4_000); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("backoff")); @@ -975,7 +1088,7 @@ mod tests { reason: Some("backoff".to_string()), }; - let state = build_recovery_state(Some(&existing), None, 10_000, None, 5_000); + let state = build_recovery_state(Some(&existing), None, true, None, 10_000, None, 5_000); assert_eq!(state.attempt, MAX_AUTO_RETRY_ATTEMPTS + 1); assert_eq!(state.gave_up_at, Some(10_000)); @@ -985,8 +1098,15 @@ mod tests { #[test] fn marks_missing_agent_session_as_manual_only() { - let state = - build_recovery_state(None, None, 10_000, Some("missing_agent_session_id"), 5_000); + let state = build_recovery_state( + None, + None, + true, + None, + 10_000, + Some("missing_agent_session_id"), + 5_000, + ); assert!(state.unsafe_to_auto_resume); assert_eq!(state.reason.as_deref(), Some("missing_agent_session_id")); @@ -1390,6 +1510,7 @@ mod tests { assert!(pause_for_rate_limit( &store, + None, &session.id, Some("rate limit"), Some("2026-06-24T15:45:00Z"), @@ -1444,10 +1565,16 @@ mod tests { ) .unwrap(); - assert!( - !pause_for_rate_limit(&store, &session.id, Some("rate limit"), None, None, 10_000) - .unwrap() - ); + assert!(!pause_for_rate_limit( + &store, + None, + &session.id, + Some("rate limit"), + None, + None, + 10_000 + ) + .unwrap()); let row = store.get_session(&session.id).unwrap().unwrap(); assert_eq!(row.status, SessionStatus::Cancelled); @@ -1543,10 +1670,12 @@ mod tests { ); } - /// A running session parked the way the runner parks a network failure. + /// A running session parked the way the runner parks a network failure, + /// with no gate to report to. fn pause_for_network(store: &Store, session_id: &str) { assert!(pause_for_rate_limit( store, + None, session_id, Some("Can't reach the API server"), None, @@ -1567,7 +1696,7 @@ mod tests { } #[test] - fn a_network_pause_is_scheduled_under_the_existing_backoff() { + fn a_network_pause_waits_for_the_gate_without_spending_an_attempt() { let store = Arc::new(Store::in_memory().unwrap()); let (_branch, session) = branch_with_running_session(&store); pause_for_network(&store, &session.id); @@ -1576,11 +1705,39 @@ mod tests { assert_eq!(listed.len(), 1); assert_eq!(listed[0].rate_limit_kind.as_deref(), Some("network")); let state = auto_recovery_from_metadata(listed[0].provider_metadata.as_ref()).unwrap(); - assert_eq!(state.attempt, 1); - let next_retry_at = state.next_retry_at.expect("a retry is scheduled"); - assert!((10_000 + MIN_RETRY_DELAY_MS..=10_000 + BASE_BACKOFF_MS).contains(&next_retry_at)); - assert!(!is_due(&state, 10_000)); - assert!(is_due(&state, next_retry_at)); + assert_eq!(state.attempt, 0); + assert_eq!(state.next_retry_at, Some(10_000)); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + assert!(is_due(&state, 10_000)); + } + + /// `counted` belongs to network pauses: an API-limit pause advances its + /// attempt whatever it is given. + #[test] + fn an_api_limit_pause_is_always_counted() { + for counted in [true, false] { + let state = + build_recovery_state(None, Some("claude_pro"), counted, None, 10_000, None, 5_000); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + } + } + + /// A network pause with an unanswered tool call is manual-only whether or + /// not it is counted. + #[test] + fn an_unsafe_network_pause_stays_manual() { + let state = build_recovery_state( + None, + Some(NETWORK_PAUSE_KIND), + false, + None, + 10_000, + Some("pending_tool_call"), + 5_000, + ); + assert!(state.unsafe_to_auto_resume); + assert_eq!(state.next_retry_at, None); } #[test] @@ -1685,18 +1842,18 @@ mod tests { // millennia out). Waiting for it would park the session forever: the // 24h give-up is only evaluated when recovery state is rebuilt. let micros = (parse_rfc3339_millis("2026-06-24T15:45:00Z").unwrap() * 1_000).to_string(); - let state = build_recovery_state(None, Some(µs), now, None, 12_000); + let state = build_recovery_state(None, None, true, Some(µs), now, None, 12_000); assert_eq!(state.reason.as_deref(), Some("backoff")); assert_eq!(jitter_bound_for(Some(µs), now), MAX_BACKOFF_MS); // The cap is on lead time, not scale: just beyond the window falls // back, just within it is honored. let beyond_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS + 60_000).to_string(); - let state = build_recovery_state(None, Some(&beyond_window), now, None, 12_000); + let state = build_recovery_state(None, None, true, Some(&beyond_window), now, None, 12_000); assert_eq!(state.reason.as_deref(), Some("backoff")); let within_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS - 60_000).to_string(); - let state = build_recovery_state(None, Some(&within_window), now, None, 12_000); + let state = build_recovery_state(None, None, true, Some(&within_window), now, None, 12_000); assert_eq!(state.reason.as_deref(), Some("reset_at")); assert_eq!(jitter_bound_for(Some(&within_window), now), RESET_JITTER_MS); } @@ -1814,7 +1971,7 @@ mod tests { // The failed attempt re-parked the session as paused; one failure // burns an attempt, not the whole recovery. - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); let state = auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap(); @@ -1845,7 +2002,7 @@ mod tests { ) .unwrap(); - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); let state = auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap(); @@ -1860,7 +2017,7 @@ mod tests { store.create_session(&session).unwrap(); // Still running: a manual resume won the race after the failed start. - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); assert_eq!( @@ -1904,7 +2061,7 @@ mod tests { ticket.rate_limit_kind.as_deref(), ) .unwrap()); - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let stopped = store.get_session(&session.id).unwrap().unwrap(); assert_eq!(stopped.status, SessionStatus::Cancelled); diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs new file mode 100644 index 000000000..0e08168b1 --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs @@ -0,0 +1,259 @@ +//! The shared gate that every network-paused session waits behind. +//! +//! A session paused with kind [`NETWORK_PAUSE_KIND`](super::NETWORK_PAUSE_KIND) +//! failed because the agent could not reach its model provider. That is a +//! fact about the machine, not about the session, so all such sessions share +//! one gate instead of each running its own backoff. API-limit pauses never +//! pass through it. +//! +//! | State | Meaning | What the scheduler does | +//! | ------- | ------------------------------- | -------------------------------- | +//! | Closed | The network is presumed down | Starts one probe when it is due | +//! | Probing | One probe turn is running | Starts no other network resume | +//! | Open | A probe produced agent output | Resumes the rest, two per tick | +//! +//! The gate lives in memory, owned by [`RateLimitScheduler`] behind a mutex, +//! and starts closed with a probe due at once, so a relaunch probes on its +//! first tick. +//! +//! Failures while the gate is not open are not counted: the session's +//! `attempt` does not advance and the 24 hour window is not applied, so a +//! laptop that sleeps for days gives up on nothing. Failures while it is open +//! are counted under the existing cap, and close it again. +//! +//! Every method takes `now`; the gate never reads a clock of its own. + +use crate::store::{SessionStatus, Store}; + +use super::RateLimitScheduler; + +/// The delay before the first probe after the gate closes, and the start of +/// the doubling for probes that fail. +pub(super) const BASE_PROBE_DELAY_MS: i64 = 15_000; +pub(super) const MAX_PROBE_DELAY_MS: i64 = 10 * 60_000; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum GateState { + Closed, + /// `session_id` was claimed as the probe at `since`. + Probing { + session_id: String, + since: i64, + }, + Open, +} + +/// What the gate lets a due network-paused session do on this tick. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum NetworkTurn { + /// Start nothing. + Hold, + /// Resume this session as the probe. + Probe, + /// Resume normally: the gate is open. + Resume, +} + +#[derive(Debug)] +pub(super) struct NetworkGate { + state: GateState, + /// When a closed gate may start its next probe. + next_probe_at: i64, + /// The wait that the next failed probe adds before the one after it. + probe_delay_ms: i64, +} + +impl Default for NetworkGate { + fn default() -> Self { + Self { + state: GateState::Closed, + next_probe_at: 0, + probe_delay_ms: BASE_PROBE_DELAY_MS, + } + } +} + +impl NetworkGate { + pub(super) fn turn(&self, now: i64) -> NetworkTurn { + match self.state { + GateState::Open => NetworkTurn::Resume, + GateState::Probing { .. } => NetworkTurn::Hold, + GateState::Closed if now >= self.next_probe_at => NetworkTurn::Probe, + GateState::Closed => NetworkTurn::Hold, + } + } + + fn probe_started(&mut self, session_id: &str, now: i64) { + self.state = GateState::Probing { + session_id: session_id.to_string(), + since: now, + }; + } + + fn probe(&self) -> Option<(String, i64)> { + match &self.state { + GateState::Probing { session_id, since } => Some((session_id.clone(), *since)), + _ => None, + } + } + + fn is_probing(&self, session_id: &str) -> bool { + matches!(&self.state, GateState::Probing { session_id: probe, .. } if probe == session_id) + } + + /// The probe's turn produced agent output: the network is back. + fn probe_succeeded(&mut self, session_id: &str) { + if self.is_probing(session_id) { + self.state = GateState::Open; + self.probe_delay_ms = BASE_PROBE_DELAY_MS; + } + } + + /// The probe ended without saying anything about the network: it was + /// stopped, deleted, or failed for another reason. Another session may + /// probe at once, and the delay is left alone. + fn probe_abandoned(&mut self, session_id: &str, now: i64) { + if self.is_probing(session_id) { + self.state = GateState::Closed; + self.next_probe_at = now; + } + } + + /// Record that `session_id` failed on the network, and answer whether the + /// failure counts against that session's retry budget. + fn record_failure(&mut self, session_id: &str, now: i64) -> bool { + match &self.state { + GateState::Open => { + self.state = GateState::Closed; + self.next_probe_at = now.saturating_add(BASE_PROBE_DELAY_MS); + self.probe_delay_ms = BASE_PROBE_DELAY_MS.saturating_mul(2); + true + } + GateState::Probing { + session_id: probe, .. + } if probe == session_id => { + self.state = GateState::Closed; + self.next_probe_at = now.saturating_add(self.probe_delay_ms); + self.probe_delay_ms = self + .probe_delay_ms + .saturating_mul(2) + .min(MAX_PROBE_DELAY_MS); + false + } + // A session that was already running when the network dropped, + // failing while the gate is closed or another session probes. It + // says nothing new, so the probe schedule stands. + GateState::Probing { .. } | GateState::Closed => false, + } + } +} + +/// How a running probe stands, read from the store. +#[derive(Debug, PartialEq, Eq)] +enum ProbeVerdict { + Running, + Succeeded, + Abandoned, +} + +/// A probe succeeds when its turn records any agent message, tool call or +/// prompt response after the claim. A turn that completed did get its prompt +/// response, whether or not the row for it is in the store yet. +fn probe_verdict(store: &Store, session_id: &str, since: i64) -> Result { + if store + .session_has_agent_output_since(session_id, since) + .map_err(|e| e.to_string())? + { + return Ok(ProbeVerdict::Succeeded); + } + let status = store + .get_session(session_id) + .map_err(|e| e.to_string())? + .map(|session| session.status); + Ok(match status { + Some(SessionStatus::Running) => ProbeVerdict::Running, + Some(SessionStatus::Completed) => ProbeVerdict::Succeeded, + // Stopped, deleted, ended on another error, or parked by something + // other than this process's pause path (which reports a network + // failure to the gate before it writes the pause). + _ => ProbeVerdict::Abandoned, + }) +} + +impl RateLimitScheduler { + /// What the gate lets a due network-paused session do on this tick. + pub(super) fn network_turn(&self, now: i64) -> NetworkTurn { + self.network_gate.lock().unwrap().turn(now) + } + + /// The scheduler claimed `session_id` as the probe. + pub(super) fn network_probe_started(&self, session_id: &str, now: i64) { + log::info!("[network_gate] probing with session {session_id}"); + self.network_gate + .lock() + .unwrap() + .probe_started(session_id, now); + } + + /// Move the gate out of `Probing` if the store shows how the probe went. + /// + /// Runs at the start of each tick, and before a network failure is + /// recorded so that a probe which produced output and then failed is + /// counted as a failure with the gate open. + pub(super) fn settle_network_probe(&self, store: &Store, now: i64) { + self.settle_probe(store, None, now); + } + + /// [`Self::settle_network_probe`], with the session whose network failure + /// is about to be recorded. If that session is the probe, a row that no + /// longer runs is its failure, not an abandoned probe: a probe that fails + /// to start is parked again before the gate hears of it. + fn settle_probe(&self, store: &Store, failing: Option<&str>, now: i64) { + // The store is read with the gate unlocked; the gate rechecks that it + // is still probing this session before it moves. + let Some((session_id, since)) = self.network_gate.lock().unwrap().probe() else { + return; + }; + match probe_verdict(store, &session_id, since) { + Ok(ProbeVerdict::Running) => {} + Ok(ProbeVerdict::Succeeded) => { + log::info!("[network_gate] probe session {session_id} got through; gate open"); + self.network_gate + .lock() + .unwrap() + .probe_succeeded(&session_id); + } + Ok(ProbeVerdict::Abandoned) if failing == Some(session_id.as_str()) => {} + Ok(ProbeVerdict::Abandoned) => { + log::info!("[network_gate] probe session {session_id} ended without an answer"); + self.network_gate + .lock() + .unwrap() + .probe_abandoned(&session_id, now); + } + Err(e) => { + log::warn!("[network_gate] failed to read probe session {session_id}: {e}"); + } + } + } + + /// Record that `session_id` failed on the network, and answer whether the + /// failure counts against that session's retry budget: it does only when + /// the gate was open. + pub(super) fn record_network_failure(&self, store: &Store, session_id: &str, now: i64) -> bool { + self.settle_probe(store, Some(session_id), now); + let counted = self + .network_gate + .lock() + .unwrap() + .record_failure(session_id, now); + log::info!( + "[network_gate] session {session_id} failed on the network ({}); gate closed", + if counted { "counted" } else { "not counted" } + ); + counted + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs new file mode 100644 index 000000000..731a491d4 --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs @@ -0,0 +1,408 @@ +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use super::super::{ + admit_for_resume, auto_recovery_from_metadata, claim_for_auto_resume, pause_for_rate_limit, + reschedule_after_start_failure, AutoResumeClaim, RateLimitAutoRecovery, RateLimitScheduler, + MAX_RESUMES_PER_TICK, NETWORK_PAUSE_KIND, +}; +use super::*; +use crate::store::{CompletionReason, MessageRole, Session, SessionStatus, Store}; + +const HOUR_MS: i64 = 60 * 60 * 1_000; + +/// The store stamps rows with the wall clock, and the scheduler lists paused +/// rows by that stamp. A short sleep keeps two writes in order. +fn step() { + std::thread::sleep(Duration::from_millis(2)); +} + +fn running_session(store: &Store) -> Session { + let session = Session::new_running("work", Path::new("/tmp")).with_agent("agent-1"); + store.create_session(&session).unwrap(); + session +} + +/// Park a running session the way the runner parks a network failure. +fn fail_on_network(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + step(); + assert!(pause_for_rate_limit( + store, + Some(scheduler), + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + now, + ) + .unwrap()); +} + +fn network_paused_sessions( + store: &Store, + scheduler: &RateLimitScheduler, + count: usize, + now: i64, +) -> Vec { + (0..count) + .map(|_| { + let session = running_session(store); + fail_on_network(store, scheduler, &session.id, now); + session.id + }) + .collect() +} + +/// The synchronous part of `run_tick`: settle the probe, then admit and +/// claim rows oldest pause first. Returns the sessions it started. +fn tick(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { + scheduler.settle_network_probe(store, now); + let mut started = Vec::new(); + for session in store.get_rate_limited_paused_sessions().unwrap() { + if started.len() >= MAX_RESUMES_PER_TICK { + break; + } + let Some(admission) = admit_for_resume(scheduler, store, &session, now) else { + continue; + }; + if let AutoResumeClaim::Claimed(_) = claim_for_auto_resume(store, &session.id).unwrap() { + if admission.probe { + scheduler.network_probe_started(&session.id, now); + } + started.push(session.id); + } + } + started +} + +fn recovery(store: &Store, session_id: &str) -> RateLimitAutoRecovery { + let session = store.get_session(session_id).unwrap().unwrap(); + auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap() +} + +fn agent_replies(store: &Store, session_id: &str) { + store + .add_session_message(session_id, MessageRole::Assistant, "Back online.") + .unwrap(); +} + +/// A gate opened by `sessions[0]` probing and getting an answer. +fn open_gate(scheduler: &RateLimitScheduler, store: &Arc, sessions: &[String], now: i64) { + assert_eq!(tick(scheduler, store, now), [sessions[0].clone()]); + agent_replies(store, &sessions[0]); + scheduler.settle_network_probe(store, now); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); +} + +#[test] +fn a_fresh_scheduler_starts_closed_with_an_immediate_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + assert_eq!(scheduler.network_turn(0), NetworkTurn::Probe); + + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // The first failure does not push the first probe out. + assert_eq!(tick(&scheduler, &store, now), sessions); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); +} + +#[test] +fn only_one_probe_starts_for_eleven_paused_rows() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 11, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + // While the probe runs, later ticks start nothing. + assert!(tick(&scheduler, &store, now + 15_000).is_empty()); + assert!(tick(&scheduler, &store, now + HOUR_MS).is_empty()); + assert_eq!(store.get_rate_limited_paused_sessions().unwrap().len(), 10); +} + +#[test] +fn a_failed_probe_spends_no_attempt_and_rotates_to_the_next_row() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 3, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + let failed_at = now + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.next_retry_at, Some(failed_at)); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + + // The next probe waits out the delay, then goes to the row that has been + // paused longest, which is no longer the one that just failed. + assert!(tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + let second_at = failed_at + BASE_PROBE_DELAY_MS; + assert_eq!(tick(&scheduler, &store, second_at), [sessions[1].clone()]); + + // A second failed probe doubles the delay. + fail_on_network(&store, &scheduler, &sessions[1], second_at); + assert!(tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS), + [sessions[2].clone()] + ); +} + +#[test] +fn a_probe_with_agent_output_opens_the_gate_and_the_rest_resume_two_per_tick() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + // Output from before the claim says nothing about the network now. + let first = running_session(&store); + agent_replies(&store, &first.id); + step(); + let now = crate::store::now_timestamp(); + fail_on_network(&store, &scheduler, &first.id, now); + let rest = network_paused_sessions(&store, &scheduler, 4, now); + + assert_eq!(tick(&scheduler, &store, now), [first.id.clone()]); + assert!(tick(&scheduler, &store, now).is_empty()); + + agent_replies(&store, &first.id); + assert_eq!(tick(&scheduler, &store, now), rest[..2]); + assert_eq!(tick(&scheduler, &store, now), rest[2..]); + for session_id in &rest { + assert_eq!(recovery(&store, session_id).attempt, 0); + } +} + +#[test] +fn a_probe_that_completes_opens_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + store + .update_session_status( + &sessions[0], + SessionStatus::Completed, + None, + Some(&CompletionReason::TurnComplete), + ) + .unwrap(); + + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); +} + +#[test] +fn a_stopped_probe_hands_the_probe_to_the_next_row_at_once() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + store + .update_session_status( + &sessions[0], + SessionStatus::Cancelled, + None, + Some(&CompletionReason::Interrupted), + ) + .unwrap(); + + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + // It is a probe, not a normal resume: the gate is still shut. + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); +} + +#[test] +fn a_failure_while_open_advances_the_attempt_and_closes_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 4, now); + open_gate(&scheduler, &store, &sessions, now); + assert_eq!(tick(&scheduler, &store, now), sessions[1..3]); + + let failed_at = now + 5_000; + fail_on_network(&store, &scheduler, &sessions[1], failed_at); + + let state = recovery(&store, &sessions[1]); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert!(state.next_retry_at.unwrap() > failed_at); + + // Closed: the row still waiting does not resume until a probe is due. + assert!(tick(&scheduler, &store, failed_at).is_empty()); + assert_eq!( + tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS), + [sessions[3].clone()] + ); + + // The other session resumed in the open period fails with the gate shut, + // so it is not counted. + fail_on_network(&store, &scheduler, &sessions[2], failed_at + 1_000); + assert_eq!(recovery(&store, &sessions[2]).attempt, 0); +} + +#[test] +fn a_probe_that_answers_and_then_fails_is_counted() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + assert_eq!(tick(&scheduler, &store, now), sessions); + agent_replies(&store, &sessions[0]); + // No tick has seen the output yet; the pause path settles the probe. + fail_on_network(&store, &scheduler, &sessions[0], now + 1_000); + + assert_eq!(recovery(&store, &sessions[0]).attempt, 1); + assert_eq!(scheduler.network_turn(now + 1_000), NetworkTurn::Hold); +} + +#[test] +fn a_row_past_24_hours_still_resumes() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // Two failed probes a day apart: neither is counted, and neither gives up. + let mut at = now; + for _ in 0..2 { + at += 25 * HOUR_MS; + assert_eq!(tick(&scheduler, &store, at), sessions); + fail_on_network(&store, &scheduler, &sessions[0], at); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.first_limited_at, now); + } + assert_eq!(tick(&scheduler, &store, at + 25 * HOUR_MS), sessions); +} + +#[test] +fn api_limit_rows_are_unaffected_by_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let network = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [network[0].clone()]); + + // An API limit while a probe runs: counted as ever, and no word to the + // gate. + let limited = running_session(&store); + step(); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + now, + ) + .unwrap()); + let state = recovery(&store, &limited.id); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); + + // It resumes on its own schedule, past the network row the gate holds. + let due = state.next_retry_at.unwrap(); + assert!(tick(&scheduler, &store, due - 1).is_empty()); + assert_eq!(tick(&scheduler, &store, due), [limited.id.clone()]); + assert_eq!(scheduler.network_turn(due), NetworkTurn::Hold); + + // Nor does an API limit close an open gate. + agent_replies(&store, &network[0]); + scheduler.settle_network_probe(&store, due); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + due, + ) + .unwrap()); + assert_eq!(recovery(&store, &limited.id).attempt, 2); + assert_eq!(scheduler.network_turn(due), NetworkTurn::Resume); +} + +#[test] +fn a_probe_that_fails_to_start_closes_the_gate_uncounted() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + assert_eq!(tick(&scheduler, &store, now), sessions); + + // The failed start re-parks the row, then reschedules it. + assert!(store + .transition_running_to_rate_limited( + &sessions[0], + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + ) + .unwrap()); + reschedule_after_start_failure(&store, &scheduler, &sessions[0], now); + + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); + assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS), + sessions + ); +} + +#[test] +fn the_probe_delay_doubles_to_a_ten_minute_cap() { + let mut gate = NetworkGate::default(); + let mut now = 1_000; + let mut delays = Vec::new(); + for _ in 0..8 { + assert_eq!(gate.turn(now), NetworkTurn::Probe); + gate.probe_started("probe", now); + assert!(!gate.record_failure("probe", now)); + delays.push(gate.next_probe_at - now); + now = gate.next_probe_at; + } + assert_eq!( + delays, + [15_000, 30_000, 60_000, 120_000, 240_000, 480_000, 600_000, 600_000] + ); + + // An open period starts the doubling again. + gate.probe_started("probe", now); + gate.probe_succeeded("probe"); + assert!(gate.record_failure("other", now)); + assert_eq!(gate.next_probe_at - now, BASE_PROBE_DELAY_MS); + now = gate.next_probe_at; + gate.probe_started("probe", now); + assert!(!gate.record_failure("probe", now)); + assert_eq!(gate.next_probe_at - now, 2 * BASE_PROBE_DELAY_MS); +} + +#[test] +fn another_sessions_failure_leaves_a_running_probe_alone() { + let mut gate = NetworkGate::default(); + gate.probe_started("probe", 1_000); + + assert!(!gate.record_failure("other", 2_000)); + assert!(gate.is_probing("probe")); + + // Nor does a stale answer about a session that is not the probe move it. + gate.probe_succeeded("other"); + gate.probe_abandoned("other", 2_000); + assert!(gate.is_probing("probe")); +} diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index 4416af72d..51959ca41 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -3485,8 +3485,14 @@ fn finish_session_run( // Status flip and retry bookkeeping land in one write, so a // scheduler tick can never see a paused row without bookkeeping // and double-count the attempt. + use tauri::Manager; + let rate_limit_scheduler = + app_handle.try_state::>(); crate::rate_limit_scheduler::pause_for_rate_limit( &store_for_status, + rate_limit_scheduler + .as_deref() + .map(|scheduler| &**scheduler), &session_id_for_status, error_msg.as_deref(), recovery.resets_at.as_deref(), diff --git a/apps/staged/src-tauri/src/session_runner/failed_run.rs b/apps/staged/src-tauri/src/session_runner/failed_run.rs index f1f54b275..ec15dffcf 100644 --- a/apps/staged/src-tauri/src/session_runner/failed_run.rs +++ b/apps/staged/src-tauri/src/session_runner/failed_run.rs @@ -168,6 +168,7 @@ mod tests { let recovery = decision.recovery.expect("a network failure pauses"); assert!(crate::rate_limit_scheduler::pause_for_rate_limit( &store, + None, &session.id, Some("Can't reach the API server"), recovery.resets_at.as_deref(), @@ -190,8 +191,11 @@ mod tests { parked.provider_metadata.as_ref(), ) .expect("the pause writes retry bookkeeping"); - assert_eq!(bookkeeping.attempt, 1); - assert!(bookkeeping.next_retry_at.is_some()); + // With the network gate not open, the pause spends no attempt and + // waits for the gate. + assert_eq!(bookkeeping.attempt, 0); + assert_eq!(bookkeeping.next_retry_at, Some(NOW_MS)); + assert_eq!(bookkeeping.reason.as_deref(), Some("network_wait")); assert!(!bookkeeping.unsafe_to_auto_resume); } diff --git a/apps/staged/src-tauri/src/session_runner/late_stop.rs b/apps/staged/src-tauri/src/session_runner/late_stop.rs index 891fca9ce..f9a9b53f5 100644 --- a/apps/staged/src-tauri/src/session_runner/late_stop.rs +++ b/apps/staged/src-tauri/src/session_runner/late_stop.rs @@ -218,6 +218,7 @@ mod tests { registry.mark_finished(&session.id, epoch); assert!(crate::rate_limit_scheduler::pause_for_rate_limit( &store, + None, &session.id, Some("rate limit"), resets_at, diff --git a/apps/staged/src-tauri/src/store/messages.rs b/apps/staged/src-tauri/src/store/messages.rs index 7922b6358..1dbe4f08c 100644 --- a/apps/staged/src-tauri/src/store/messages.rs +++ b/apps/staged/src-tauri/src/store/messages.rs @@ -18,6 +18,16 @@ const VISIBLE_MESSAGE_FILTER: &str = "NOT (content = '' AND acp_event_kind IS NO /// button compares against. pub(super) const VISIBLE_ASSISTANT_FILTER: &str = "role = 'assistant' AND NOT (content = '' AND acp_event_kind IS NOT NULL)"; +/// A row of the agent's own output for a turn: text, a tool call or its +/// result, a plan, or the turn's response. Hidden rows tagged with an +/// `acp_origin` and the rows a connection writes before it sends the prompt +/// are left out. +const AGENT_OUTPUT_FILTER: &str = "acp_origin IS NULL + AND (role IN ('tool_call', 'tool_result') + OR (role = 'assistant' + AND (acp_event_kind IS NULL + OR acp_event_kind IN ('agent_message_chunk', 'plan_update', + 'response_result', 'prompt_response'))))"; /// ACP rows the transcript needs on top of [`Store::get_session_messages`]: /// hidden rows of the kinds the frontend reads (`acpTranscript.ts`, /// `acpConfigMetadata.ts`), plus visible tool rows still in flight — the @@ -679,20 +689,42 @@ impl Store { ) -> Result { let conn = self.conn.lock().unwrap(); conn.query_row( - "SELECT EXISTS( - SELECT 1 FROM session_messages - WHERE session_id = ?1 AND id > ?2 AND acp_origin IS NULL - AND (role IN ('tool_call', 'tool_result') - OR (role = 'assistant' - AND (acp_event_kind IS NULL - OR acp_event_kind IN ('agent_message_chunk', 'plan_update', - 'response_result', 'prompt_response')))))", + &format!( + "SELECT EXISTS( + SELECT 1 FROM session_messages + WHERE session_id = ?1 AND id > ?2 AND {AGENT_OUTPUT_FILTER})" + ), params![session_id, message_id], |row| row.get(0), ) .map_err(Into::into) } + /// Whether a session recorded agent output at or after `since`: text, a + /// tool call or its result, a plan, or a turn's response. The rows counted + /// are those of [`Self::agent_answered_message`], bounded by time instead + /// of by a message id. + /// + /// Backs the network gate's probe check, which knows when it claimed the + /// probe but not which row its prompt will be. + pub fn session_has_agent_output_since( + &self, + session_id: &str, + since: i64, + ) -> Result { + let conn = self.conn.lock().unwrap(); + conn.query_row( + &format!( + "SELECT EXISTS( + SELECT 1 FROM session_messages + WHERE session_id = ?1 AND created_at >= ?2 AND {AGENT_OUTPUT_FILTER})" + ), + params![session_id, since], + |row| row.get(0), + ) + .map_err(Into::into) + } + /// Count assistant messages created after a given timestamp. pub fn count_assistant_messages_after( &self, From d8c535eef05a8d3452c45c252943b512164e9b38 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 15:46:54 +1100 Subject: [PATCH 5/8] feat(staged): probe the network gate on wake and network change Step 5 of the network pause plan: a closed network gate no longer waits out its doubling delay when the machine wakes or its network changes. Both triggers are derived inside the scheduler tick, with no operating-system callbacks. - Wake. The gate records the wall-clock time of each tick. A gap of more than 45 seconds on the 15 second tick means the process was suspended, and the probe delay resets so a probe is due at once. - Settle rule. A probe starts only on a tick whose preceding gap was normal. A short maintenance wake cannot produce two ticks 15 seconds apart, so it never starts a probe; after the lid opens, the second tick does. The first tick of a process has no gap and still probes. - Network change. While the gate is closed and a network-paused row could still resume by itself, each tick reads the machine's interface addresses. A change resets the probe delay. An empty set means no network, and no probe starts; the delay does not advance meanwhile. The reset sets the delay back to 15 seconds and, for a closed gate, makes the next probe due now. A running probe is left to finish. The first read of a closed period is the baseline, not a change, and the baseline is dropped when the gate opens. A failed read clears a "no network" answer, so it cannot block probes. The addresses come through the new `NetworkFingerprint` trait in `network_gate/fingerprint.rs`. The real implementation uses the new `if-addrs` dependency, flagged for review in the plan. It leaves out loopback addresses and, beyond the plan's wording, link-local ones: macOS keeps an `fe80::` address on its `utun` and `awdl` interfaces with no network, so counting them would make the set never empty. `RateLimitScheduler` owns the fingerprint and takes a fake through `with_network_fingerprint`. `run_tick` calls `observe_network_tick` after it settles the probe and before it admits any row. Tests drive the synchronous part of the tick, triggers included, with an injected clock and a fake fingerprint: a long gap resets the probe delay; no probe starts on the tick after a long gap; a gap of exactly 45 seconds is normal; a run of long gaps never starts a probe; a fingerprint change resets the delay; an empty fingerprint blocks the probe without advancing the delay; losing every address blocks a due probe; an unreadable fingerprint does not block; the fingerprint is read only while the gate is closed with network rows; and the real fingerprint holds no loopback or link-local address. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- apps/staged/src-tauri/Cargo.lock | 11 + apps/staged/src-tauri/Cargo.toml | 3 + .../src-tauri/src/rate_limit_scheduler.rs | 34 +- .../src/rate_limit_scheduler/network_gate.rs | 137 +++++++- .../network_gate/fingerprint.rs | 34 ++ .../network_gate/tests.rs | 28 +- .../network_gate/tests/triggers.rs | 309 ++++++++++++++++++ 7 files changed, 547 insertions(+), 9 deletions(-) create mode 100644 apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs create mode 100644 apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs diff --git a/apps/staged/src-tauri/Cargo.lock b/apps/staged/src-tauri/Cargo.lock index d81845a0d..2ca5bb72c 100644 --- a/apps/staged/src-tauri/Cargo.lock +++ b/apps/staged/src-tauri/Cargo.lock @@ -20,6 +20,7 @@ dependencies = [ "flate2", "git-diff", "hex", + "if-addrs", "image", "imagesize", "include_dir", @@ -2630,6 +2631,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "image" version = "0.25.10" diff --git a/apps/staged/src-tauri/Cargo.toml b/apps/staged/src-tauri/Cargo.toml index 691abe855..569ec008d 100644 --- a/apps/staged/src-tauri/Cargo.toml +++ b/apps/staged/src-tauri/Cargo.toml @@ -21,6 +21,9 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } log = "0.4" libc = "0.2" +# Network interface addresses, read by the rate-limit scheduler's network +# gate to notice a network change while sessions are paused on the network. +if-addrs = "0.15" tauri = { version = "2.12.1", features = [] } tauri-plugin-log = "2.10.0" diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index 263cf34b2..e625f721f 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -25,7 +25,7 @@ use crate::store::{MessageRole, Session, Store}; mod network_gate; -use network_gate::{NetworkGate, NetworkTurn}; +use network_gate::{InterfaceAddresses, NetworkFingerprint, NetworkGate, NetworkTurn}; pub(crate) const AUTO_RETRY_RESUME_PROMPT: &str = "Continue where you left off."; /// The resume prompt for a session paused by a network failure. The agent's @@ -75,17 +75,35 @@ const MAX_RESUMES_PER_TICK: usize = 2; /// to plain backoff instead. pub(crate) const MAX_PLAUSIBLE_RESET_LEAD_MS: i64 = 48 * 60 * 60 * 1_000; -#[derive(Default)] pub struct RateLimitScheduler { in_flight: Mutex>, tick_loop_spawned: AtomicBool, /// The gate every network-paused session waits behind. network_gate: Mutex, + /// Where the gate reads the machine's interface addresses from. + network_fingerprint: Box, +} + +impl Default for RateLimitScheduler { + fn default() -> Self { + Self::new() + } } impl RateLimitScheduler { pub fn new() -> Self { - Self::default() + Self::with_network_fingerprint(Box::new(InterfaceAddresses)) + } + + pub(crate) fn with_network_fingerprint( + network_fingerprint: Box, + ) -> Self { + Self { + in_flight: Mutex::default(), + tick_loop_spawned: AtomicBool::default(), + network_gate: Mutex::default(), + network_fingerprint, + } } fn try_mark_in_flight(&self, session_id: &str) -> bool { @@ -176,6 +194,7 @@ async fn run_tick( let mut started = 0usize; let mut drained_branches: HashSet = HashSet::new(); scheduler.settle_network_probe(&store, now); + scheduler.observe_network_tick(now, network_rows_waiting(&sessions)); for session in sessions { if started >= MAX_RESUMES_PER_TICK { @@ -374,6 +393,15 @@ fn admit_for_resume( Some(ResumeAdmission { state, probe }) } +/// Whether any listed session is paused on the network and may still resume +/// by itself: the rows the network gate could probe with. +fn network_rows_waiting(sessions: &[Session]) -> bool { + sessions.iter().any(|session| { + is_network_pause(session.rate_limit_kind.as_deref()) + && is_actionable_for_auto_resume(session) + }) +} + fn is_network_pause(rate_limit_kind: Option<&str>) -> bool { rate_limit_kind == Some(NETWORK_PAUSE_KIND) } diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs index 0e08168b1..29172d0e8 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs @@ -21,16 +21,39 @@ //! laptop that sleeps for days gives up on nothing. Failures while it is open //! are counted under the existing cap, and close it again. //! +//! Two things make a closed gate probe sooner than its doubling delay, both +//! derived inside the scheduler tick with no operating-system callbacks: +//! +//! - **Wake.** A gap of more than [`WAKE_GAP_MS`] between two ticks means the +//! process was suspended. The probe delay resets, so a probe is due at once. +//! - **Network change.** While the gate is closed, each tick reads the +//! machine's [`NetworkFingerprint`]. A change resets the probe delay. An +//! empty set means no network, and no probe starts. +//! +//! A probe starts only on a tick whose preceding gap was normal. A short +//! maintenance wake during sleep cannot produce two ticks one interval apart, +//! so it never starts a probe; after the lid opens, the second tick does. +//! //! Every method takes `now`; the gate never reads a clock of its own. +use std::collections::BTreeSet; +use std::net::IpAddr; + use crate::store::{SessionStatus, Store}; use super::RateLimitScheduler; +mod fingerprint; + +pub(crate) use fingerprint::{InterfaceAddresses, NetworkFingerprint}; + /// The delay before the first probe after the gate closes, and the start of /// the doubling for probes that fail. pub(super) const BASE_PROBE_DELAY_MS: i64 = 15_000; pub(super) const MAX_PROBE_DELAY_MS: i64 = 10 * 60_000; +/// A gap between two ticks longer than this means the process was suspended: +/// three of the scheduler's 15 second ticks. +pub(super) const WAKE_GAP_MS: i64 = 45_000; #[derive(Debug, Clone, PartialEq, Eq)] enum GateState { @@ -61,6 +84,14 @@ pub(super) struct NetworkGate { next_probe_at: i64, /// The wait that the next failed probe adds before the one after it. probe_delay_ms: i64, + /// The wall-clock time of the last tick. + last_tick_at: Option, + /// Whether the gap before the last tick was normal. A probe starts only + /// when it was; the first tick of a process has no gap and counts. + settled: bool, + /// The fingerprint last read while the gate was closed. `None` until the + /// first read, after the gate opens, and when a read fails. + addresses: Option>, } impl Default for NetworkGate { @@ -69,6 +100,9 @@ impl Default for NetworkGate { state: GateState::Closed, next_probe_at: 0, probe_delay_ms: BASE_PROBE_DELAY_MS, + last_tick_at: None, + settled: true, + addresses: None, } } } @@ -78,11 +112,72 @@ impl NetworkGate { match self.state { GateState::Open => NetworkTurn::Resume, GateState::Probing { .. } => NetworkTurn::Hold, - GateState::Closed if now >= self.next_probe_at => NetworkTurn::Probe, + GateState::Closed + if now >= self.next_probe_at && self.settled && !self.network_absent() => + { + NetworkTurn::Probe + } GateState::Closed => NetworkTurn::Hold, } } + /// Record a scheduler tick, and answer whether the gap before it shows + /// that the process was suspended. + fn observe_tick(&mut self, now: i64) -> bool { + let woke = self + .last_tick_at + .is_some_and(|last| now.saturating_sub(last) > WAKE_GAP_MS); + self.last_tick_at = Some(now); + self.settled = !woke; + if woke { + self.reset_probe_delay(now); + } + woke + } + + /// Something suggests the network may be back. A closed gate may probe + /// at once, and the next failed probe waits the base delay again. A + /// running probe is left to finish. + fn reset_probe_delay(&mut self, now: i64) { + self.probe_delay_ms = BASE_PROBE_DELAY_MS; + if self.state == GateState::Closed { + self.next_probe_at = now; + } + } + + fn is_closed(&self) -> bool { + self.state == GateState::Closed + } + + /// The last fingerprint read held no address: there is no network. + fn network_absent(&self) -> bool { + self.addresses.as_ref().is_some_and(BTreeSet::is_empty) + } + + /// Record the fingerprint read on this tick, and answer whether it + /// differs from the one before. The first read is the baseline, not a + /// change. + fn observe_fingerprint(&mut self, addresses: BTreeSet, now: i64) -> bool { + if !self.is_closed() { + return false; + } + let changed = self + .addresses + .as_ref() + .is_some_and(|previous| *previous != addresses); + self.addresses = Some(addresses); + if changed { + self.reset_probe_delay(now); + } + changed + } + + /// The fingerprint could not be read. Nothing is known about the + /// network, so an earlier "no network" answer no longer blocks a probe. + fn fingerprint_unreadable(&mut self) { + self.addresses = None; + } + fn probe_started(&mut self, session_id: &str, now: i64) { self.state = GateState::Probing { session_id: session_id.to_string(), @@ -106,6 +201,9 @@ impl NetworkGate { if self.is_probing(session_id) { self.state = GateState::Open; self.probe_delay_ms = BASE_PROBE_DELAY_MS; + // The fingerprint is not read while the gate is open, so the + // next closed period starts from a fresh baseline. + self.addresses = None; } } @@ -186,6 +284,43 @@ impl RateLimitScheduler { self.network_gate.lock().unwrap().turn(now) } + /// Apply the wake and network-change triggers for the tick at `now`. + /// + /// Runs once per tick, before any row is admitted. The fingerprint is + /// read only while the gate is closed and `network_rows_waiting`, so a + /// machine with nothing paused on the network never lists its interfaces. + pub(super) fn observe_network_tick(&self, now: i64, network_rows_waiting: bool) { + let closed = { + let mut gate = self.network_gate.lock().unwrap(); + if gate.observe_tick(now) { + log::info!("[network_gate] woke from a suspend; probe delay reset"); + } + gate.is_closed() + }; + if !closed || !network_rows_waiting { + return; + } + // The interfaces are read with the gate unlocked; the gate rechecks + // that it is still closed before it takes the answer. + let addresses = self.network_fingerprint.read(); + let mut gate = self.network_gate.lock().unwrap(); + match addresses { + Ok(addresses) => { + let was_absent = gate.network_absent(); + let changed = gate.observe_fingerprint(addresses, now); + if gate.network_absent() && !was_absent { + log::info!("[network_gate] no network interface has an address; not probing"); + } else if changed { + log::info!("[network_gate] network interfaces changed; probe delay reset"); + } + } + Err(e) => { + log::warn!("[network_gate] failed to read network interfaces: {e}"); + gate.fingerprint_unreadable(); + } + } + } + /// The scheduler claimed `session_id` as the probe. pub(super) fn network_probe_started(&self, session_id: &str, now: i64) { log::info!("[network_gate] probing with session {session_id}"); diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs new file mode 100644 index 000000000..cb986515b --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs @@ -0,0 +1,34 @@ +//! The machine's network fingerprint: the set of addresses its interfaces +//! hold. The gate reads it while closed to notice that the network changed, +//! or that there is none. + +use std::collections::BTreeSet; +use std::io; +use std::net::IpAddr; + +/// Reads the set of interface addresses that could carry a request off this +/// machine. An empty set means there is no network. +pub(crate) trait NetworkFingerprint: Send + Sync { + fn read(&self) -> io::Result>; +} + +/// The real fingerprint, read from the operating system's interface list. +pub(crate) struct InterfaceAddresses; + +impl NetworkFingerprint for InterfaceAddresses { + fn read(&self) -> io::Result> { + Ok(if_addrs::get_if_addrs()? + .into_iter() + .filter(counts_as_network) + .map(|interface| interface.ip()) + .collect()) + } +} + +/// Loopback addresses never leave the machine. Link-local ones are left out +/// too: macOS keeps an `fe80::` address on its `utun` and `awdl` interfaces +/// with Wi-Fi off and no cable, so counting them would make the set never +/// empty and "no network" undetectable. +fn counts_as_network(interface: &if_addrs::Interface) -> bool { + !interface.is_loopback() && !interface.is_link_local() +} diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs index 731a491d4..8b5b9e47b 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs @@ -3,11 +3,13 @@ use std::sync::Arc; use std::time::Duration; use super::super::{ - admit_for_resume, auto_recovery_from_metadata, claim_for_auto_resume, pause_for_rate_limit, - reschedule_after_start_failure, AutoResumeClaim, RateLimitAutoRecovery, RateLimitScheduler, - MAX_RESUMES_PER_TICK, NETWORK_PAUSE_KIND, + admit_for_resume, auto_recovery_from_metadata, claim_for_auto_resume, network_rows_waiting, + pause_for_rate_limit, reschedule_after_start_failure, AutoResumeClaim, RateLimitAutoRecovery, + RateLimitScheduler, MAX_RESUMES_PER_TICK, NETWORK_PAUSE_KIND, }; use super::*; + +mod triggers; use crate::store::{CompletionReason, MessageRole, Session, SessionStatus, Store}; const HOUR_MS: i64 = 60 * 60 * 1_000; @@ -54,10 +56,26 @@ fn network_paused_sessions( .collect() } -/// The synchronous part of `run_tick`: settle the probe, then admit and -/// claim rows oldest pause first. Returns the sessions it started. +/// The synchronous part of `run_tick` without the wake and network-change +/// triggers: settle the probe, then admit and claim rows oldest pause first. +/// Returns the sessions it started. +/// +/// The tests in this file jump the clock freely to exercise the gate's own +/// rules; `triggers.rs` drives [`tick_with_triggers`] on a real cadence. fn tick(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { scheduler.settle_network_probe(store, now); + admit_and_claim(scheduler, store, now) +} + +/// The whole synchronous part of `run_tick`, triggers included. +fn tick_with_triggers(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { + let sessions = store.get_rate_limited_paused_sessions().unwrap(); + scheduler.settle_network_probe(store, now); + scheduler.observe_network_tick(now, network_rows_waiting(&sessions)); + admit_and_claim(scheduler, store, now) +} + +fn admit_and_claim(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { let mut started = Vec::new(); for session in store.get_rate_limited_paused_sessions().unwrap() { if started.len() >= MAX_RESUMES_PER_TICK { diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs new file mode 100644 index 000000000..cab3003fb --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs @@ -0,0 +1,309 @@ +//! The wake and network-change triggers, driven tick by tick with an +//! injected clock and a fake fingerprint. + +use std::collections::BTreeSet; +use std::io; +use std::net::IpAddr; +use std::sync::Mutex; + +use super::*; + +/// The scheduler's tick interval. +const TICK_MS: i64 = 15_000; +/// A gap between ticks that only a suspended process produces. +const SUSPENDED_MS: i64 = WAKE_GAP_MS + 1; + +#[derive(Default)] +struct FakeNetworkState { + /// `None` makes the read fail. + addresses: Option>, + reads: usize, +} + +/// A fingerprint the test sets by hand, which counts how often it is read. +#[derive(Clone)] +struct FakeNetwork(Arc>); + +impl FakeNetwork { + fn with_addresses(addresses: &[&str]) -> Self { + let network = Self(Arc::default()); + network.set_addresses(addresses); + network + } + + fn set_addresses(&self, addresses: &[&str]) { + self.0.lock().unwrap().addresses = Some( + addresses + .iter() + .map(|address| address.parse().unwrap()) + .collect(), + ); + } + + fn make_unreadable(&self) { + self.0.lock().unwrap().addresses = None; + } + + fn reads(&self) -> usize { + self.0.lock().unwrap().reads + } + + fn scheduler(&self) -> RateLimitScheduler { + RateLimitScheduler::with_network_fingerprint(Box::new(self.clone())) + } +} + +impl NetworkFingerprint for FakeNetwork { + fn read(&self) -> io::Result> { + let mut state = self.0.lock().unwrap(); + state.reads += 1; + state + .addresses + .clone() + .ok_or_else(|| io::Error::other("interfaces unavailable")) + } +} + +const WIFI: &[&str] = &["192.0.2.10"]; +const OTHER_WIFI: &[&str] = &["198.51.100.7"]; + +/// Tick every 15 seconds from `from`, failing each probe as it starts, until +/// four have failed. The fourth fails 105 seconds in and leaves the next +/// probe two minutes away. Returns the time of the last tick. +fn fail_four_probes(scheduler: &RateLimitScheduler, store: &Arc, from: i64) -> i64 { + let mut at = from; + let mut failures = 0; + loop { + for session_id in tick_with_triggers(scheduler, store, at) { + fail_on_network(store, scheduler, &session_id, at); + failures += 1; + } + if failures == 4 { + assert_eq!(at, from + 7 * TICK_MS); + return at; + } + at += TICK_MS; + } +} + +#[test] +fn a_long_gap_resets_the_probe_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + // The next probe is two minutes off, so an ordinary tick starts nothing. + let before_sleep = last + TICK_MS; + assert!(tick_with_triggers(&scheduler, &store, before_sleep).is_empty()); + + // The machine sleeps. The tick that sees the gap starts nothing, and the + // one after it probes, long before the two minutes are up. + let woke = before_sleep + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); + let probed = woke + TICK_MS; + let probe = tick_with_triggers(&scheduler, &store, probed); + assert_eq!(probe.len(), 1); + + // The doubling starts over too: a failure waits the base delay. + fail_on_network(&store, &scheduler, &probe[0], probed); + assert!(tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS).len(), + 1 + ); +} + +#[test] +fn no_probe_starts_on_the_tick_after_a_long_gap() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + // A probe is due at once: the gate is fresh. + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let woke = now + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, woke + TICK_MS), + sessions + ); +} + +#[test] +fn a_gap_of_exactly_the_threshold_is_normal() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + assert_eq!( + tick_with_triggers(&scheduler, &store, now + WAKE_GAP_MS), + sessions + ); +} + +#[test] +fn a_run_of_long_gaps_never_starts_a_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // Maintenance wakes during a night of sleep: one tick each, far apart. + let mut at = now; + for _ in 0..20 { + at += 20 * 60_000; + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + } + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); + + assert_eq!( + tick_with_triggers(&scheduler, &store, at + TICK_MS), + sessions + ); +} + +#[test] +fn a_fingerprint_change_resets_the_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + // The same network: the two minute wait stands. + assert!(tick_with_triggers(&scheduler, &store, last + TICK_MS).is_empty()); + + network.set_addresses(OTHER_WIFI); + let probed = last + 2 * TICK_MS; + let probe = tick_with_triggers(&scheduler, &store, probed); + assert_eq!(probe.len(), 1); + + fail_on_network(&store, &scheduler, &probe[0], probed); + assert!(tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS).len(), + 1 + ); +} + +#[test] +fn an_empty_fingerprint_blocks_the_probe_without_advancing_the_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(&[]); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + for _ in 0..40 { + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + at += TICK_MS; + } + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); + + // An address appears, and the probe starts on that tick. + network.set_addresses(WIFI); + assert_eq!(tick_with_triggers(&scheduler, &store, at), sessions); + + // Ten minutes without a network doubled nothing: this is the first + // failed probe, and the next one is the base delay away. + fail_on_network(&store, &scheduler, &sessions[0], at); + assert!(tick_with_triggers(&scheduler, &store, at + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, at + BASE_PROBE_DELAY_MS), + sessions + ); +} + +#[test] +fn losing_every_address_blocks_a_probe_that_was_due() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + assert_eq!(tick_with_triggers(&scheduler, &store, now), sessions); + fail_on_network(&store, &scheduler, &sessions[0], now); + + network.set_addresses(&[]); + assert!(tick_with_triggers(&scheduler, &store, now + TICK_MS).is_empty()); + assert!(tick_with_triggers(&scheduler, &store, now + 2 * TICK_MS).is_empty()); + + network.set_addresses(WIFI); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + 3 * TICK_MS), + sessions + ); +} + +#[test] +fn an_unreadable_fingerprint_does_not_block_the_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(&[]); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + + network.make_unreadable(); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + TICK_MS), + sessions + ); +} + +#[test] +fn the_fingerprint_is_read_only_while_closed_with_network_rows() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + // The store stamps the probe's reply with the wall clock, so the ticks + // run in the past: the reply then lands after the probe's claim. + let now = crate::store::now_timestamp() - 10 * TICK_MS; + + // Closed, but nothing is paused on the network. + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + assert_eq!(network.reads(), 0); + + // Closed with a row waiting: read, and the row probes. + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + TICK_MS), + [sessions[0].clone()] + ); + assert_eq!(network.reads(), 1); + + // Probing: not read. + assert!(tick_with_triggers(&scheduler, &store, now + 2 * TICK_MS).is_empty()); + assert_eq!(network.reads(), 1); + + // Open: not read. + agent_replies(&store, &sessions[0]); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + 3 * TICK_MS), + [sessions[1].clone()] + ); + assert_eq!(network.reads(), 1); +} + +#[test] +fn the_real_fingerprint_leaves_out_loopback_and_link_local_addresses() { + let Ok(addresses) = InterfaceAddresses.read() else { + return; + }; + for address in addresses { + assert!(!address.is_loopback(), "{address}"); + let link_local = match address { + IpAddr::V4(v4) => v4.is_link_local(), + IpAddr::V6(v6) => (v6.segments()[0] & 0xffc0) == 0xfe80, + }; + assert!(!link_local, "{address}"); + } +} From b385cb1eb7a93e47cde49ecfe31fa332b69a4deb Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 15:49:21 +1100 Subject: [PATCH 6/8] feat(staged): tell project sessions about paused children Step 6 of the network pause plan: a project session that waits on a child now learns that a paused child is not finished and needs nothing from it. Until now the `wait_for_repo_session` description named queued, running and disconnected as the states to wait on again, so a parent that saw `paused` had no guidance and could cancel or restart a child that Staged was about to resume. The description now lists `paused` with those states and says why a child is paused and what happens next: its agent hit a provider limit or lost its network connection, and Staged resumes a paused session by itself, so it needs no follow-up. The wait itself already kept waiting on `paused`, because it returns early only for a terminal state. Two comments now cover both kinds of pause. The `paused` arm of `cancel_repo_session` says the store write drops the completion reason that both kinds carry, so the scheduler never resumes the child and a network pause never probes with it. The status gate in `pipeline_dependency.rs` says a paused dependency resumes by itself for either kind. No behaviour changes in either place. The existing tool-description test now checks the wait-again clause for all four states, the two causes of a pause, and the statement that Staged resumes the session. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../src-tauri/src/pipeline_dependency.rs | 5 ++- apps/staged/src-tauri/src/project_mcp.rs | 37 ++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/staged/src-tauri/src/pipeline_dependency.rs b/apps/staged/src-tauri/src/pipeline_dependency.rs index dc1a2bf2b..f9aad6e05 100644 --- a/apps/staged/src-tauri/src/pipeline_dependency.rs +++ b/apps/staged/src-tauri/src/pipeline_dependency.rs @@ -68,8 +68,9 @@ fn status_gate(dependency: Option<&store::Session>) -> Option { let noun = dependency_noun(dependency); match dependency.status { // A disconnected dependency may still be running on its workstation, - // and a paused one hit a rate limit and resumes on its own, so neither - // has finished yet. + // and a paused one resumes on its own, whether it hit a provider + // limit or, with `rate_limit_kind = "network"`, its prompt failed on + // the network. Neither has finished yet. SessionStatus::Queued | SessionStatus::Running | SessionStatus::Disconnected diff --git a/apps/staged/src-tauri/src/project_mcp.rs b/apps/staged/src-tauri/src/project_mcp.rs index bdb0eb8d0..0bcd5dab5 100644 --- a/apps/staged/src-tauri/src/project_mcp.rs +++ b/apps/staged/src-tauri/src/project_mcp.rs @@ -827,7 +827,7 @@ impl ProjectToolsHandler { } #[tool( - description = "Wait for a repo session started by `start_repo_session`. Returns the current state and any available artifacts for the opaque `repo_session_id`; note content references attached images and videos by local file path. `wait_for_completion_seconds` defaults to 240. Calls without an MCP progress token return within 30 seconds; repeat the wait if the session is still queued or running, or disconnected (a remote session Staged lost its connection to; it may still be working). A `failed` session reports `completion_reason` and `error_message`; `remote_agent_lost` means its remote workspace agent died under it (the message names why, such as the workstation restarting), so work it left may be uncommitted on the workspace. A completed session has finished its one goal: follow up with `continue_repo_session` only to answer a question it ended on or to fix that outcome, and start a new repo session for whatever comes next." + description = "Wait for a repo session started by `start_repo_session`. Returns the current state and any available artifacts for the opaque `repo_session_id`; note content references attached images and videos by local file path. `wait_for_completion_seconds` defaults to 240. Calls without an MCP progress token return within 30 seconds; repeat the wait if the session is still queued or running, disconnected (a remote session Staged lost its connection to; it may still be working), or paused (its agent hit a provider limit or lost its network connection; Staged resumes a paused session by itself, so it needs no follow-up). A `failed` session reports `completion_reason` and `error_message`; `remote_agent_lost` means its remote workspace agent died under it (the message names why, such as the workstation restarting), so work it left may be uncommitted on the workspace. A completed session has finished its one goal: follow up with `continue_repo_session` only to answer a question it ended on or to fix that outcome, and start a new repo session for whatever comes next." )] async fn wait_for_repo_session( &self, @@ -946,11 +946,15 @@ impl ProjectToolsHandler { ); } } - // A session parked on a rate limit has no runner and no - // token: the store write is the whole cancel. It drops - // `completion_reason = rate_limited`, so the rate-limit - // scheduler never resumes the child, and settles the - // work queued behind it, as a Stop in the UI does. + // A paused session is parked for automatic resume, on a + // provider limit or, with `rate_limit_kind = "network"`, + // on a prompt that failed on the network. Either kind has + // no runner and no token: the store write is the whole + // cancel. It drops `completion_reason = rate_limited`, + // which both kinds carry, so the rate-limit scheduler + // never resumes the child and a network pause never + // probes with it, and settles the work queued behind it, + // as a Stop in the UI does. Ok(Some(session)) if session.status == SessionStatus::Paused => { crate::session_commands::cancel_session_without_runner( &self.store, @@ -1782,9 +1786,10 @@ mod tests { ); } - /// A child parked on a rate limit has no runner, so neither the queued - /// transition nor the registry can cancel it; the tool used to fall - /// through, report `paused`, and leave the scheduler to restart the child. + /// A paused child, parked on a provider limit or a network failure, has + /// no runner, so neither the queued transition nor the registry can cancel + /// it; the tool used to fall through, report `paused`, and leave the + /// scheduler to restart the child. /// The store write itself is covered by /// `session_commands::tests::cancelling_a_paused_session_without_a_runner_keeps_the_callers_reason_and_stops_auto_resume`; /// that the tool reaches it is pinned by source text, because the handler @@ -1824,6 +1829,20 @@ mod tests { .expect("cancel tool description"); assert!(wait_description.contains("Returns the current state")); + // A paused child is not finished and Staged restarts it, so the + // parent must be told to wait again, for either cause of the pause. + let repeat = &wait_description[wait_description + .find("repeat the wait if the session is still") + .expect("repeat-the-wait clause")..]; + let repeat = &repeat[..repeat.find("A `failed` session").expect("failed clause")]; + for state in ["queued", "running", "disconnected", "paused"] { + assert!( + repeat.contains(state), + "the wait description must tell the parent to wait again for `{state}`: {repeat}" + ); + } + assert!(repeat.contains("hit a provider limit or lost its network connection")); + assert!(repeat.contains("Staged resumes a paused session by itself")); assert!(!wait_description.contains("`activity` progress fields")); assert!(!wait_description.contains("`last_activity_at`")); assert!(!wait_description.contains("`last_tool_result`")); From dddf059752f5af2d45fb3f19fcc07dd690816fde Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 16:45:35 +1100 Subject: [PATCH 7/8] fix(staged): apply the network pause review Fixes every finding of the review of b385cb1e. The chief one: a failed Claude probe was judged a success, so failures that should have waited for the gate were counted and the gate could open into an outage. Probe success. A failed Claude turn records more than the error result the review named. Stored turns show the CLI's error as an `agent_message_chunk`, a `response_result` with `is_error: true` (its `subtype` still reads `success`), and the same text as the visible assistant row. All three counted as agent output. The probe check is now the store query `session_reached_provider_since`: a tool call or its result, a `response_result` without the error mark, or a `prompt_response`. Message text and plans no longer count, so a text-only probe opens the gate when its turn ends, not at its first chunk. 24 hour window. For a network pause it runs from the first counted failure: `first_limited_at` is set again when `attempt` becomes 1. No field is added. Probe cap. A session whose own probes fail `MAX_CONSECUTIVE_PROBE_FAILURES` (20) times in a row, each with the machine awake and a non-empty fingerprint, gives up with reason `probe_limit_exceeded` and stays paused for a manual resume. A probe during which the process was suspended, or made with an empty or unreadable fingerprint, neither adds to the run nor breaks it. The counts are held in memory by the gate and cleared when it opens. The gate now answers a failure with `PauseAccounting` (`Counted`, `Waiting`, `ProbesExhausted`) in place of a bool. Failed start. A resume that fails to start abandons the probe (another row may probe at once, the delay does not grow) and is counted under the existing cap of eight. Empty fingerprint. It no longer blocks a probe. It only holds back the wake and change triggers, and the doubling schedule keeps running, so a session whose provider is local is probed with no network. Probe timeout. A probe still running after `PROBE_TIMEOUT_MS` (10 minutes) without reaching its provider closes the gate as a failed probe does. The session keeps running and is not counted. Manual resume. `resume_session_for_store` tells the scheduler when its claim wins. A closed gate watches a network-paused session's turn as its probe, so a resume that gets through opens the gate for the rest. Prompt-failure check. The driver reports it as a flag: `SessionSettled::prompt_failed`, read from the transport monitor's existing `answered_error`, carried on the runner's `RunSettlement` to `decide_failed_run`. `failed_on_the_prompt` and its parsing of the SDK's Debug text are gone, and `PROMPT_FAILED_PREFIX` is private to the driver again. Nothing stored changes. A turn reattached to after its connection went has no flag and is not classified. Chat alert. A row whose bookkeeping reason is `network_wait` shows no retry time: its `nextRetryAt` is the time it paused. Row subtitle. A timeline row carries no retry bookkeeping, so it cannot tell a pause that will resume from one that gave up or is unsafe. It now reads "Paused: network unavailable." with no promise. Keeping the promise on healthy rows needs a new timeline payload field, left for review. Tests, each failing on b385cb1e: the real row sequence of a failed Claude turn leaves the gate shut and the failure uncounted, also for a tick between those rows and the pause write; a successful result or a prompt response opens the gate and text does not; a counted failure after a 30 hour outage schedules a retry, and the window still ends 24 hours after the first counted failure; twenty failed probes with a network give up, and with none never do; a failed start is counted, hands the probe over and gives up at the cap; an empty fingerprint keeps the doubling schedule and suppresses the wake and change triggers; a hung probe times out; a manual resume opens the gate, or closes it uncounted when it fails; the driver marks an answered prompt error; a run not flagged as a failed prompt is not paused; the alert for a `network_wait` row with a past time; and the subtitle without a promise. The two `cloned_ref_to_slice_refs` warnings are fixed. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../src-tauri/src/rate_limit_scheduler.rs | 311 +++++++++-- .../src/rate_limit_scheduler/network_gate.rs | 311 ++++++++--- .../network_gate/fingerprint.rs | 4 +- .../network_gate/tests.rs | 509 +++++++++++++++++- .../network_gate/tests/triggers.rs | 176 ++++-- apps/staged/src-tauri/src/session_commands.rs | 7 + apps/staged/src-tauri/src/session_runner.rs | 11 + .../src/session_runner/failed_run.rs | 62 ++- .../src/session_runner/network_failure.rs | 109 +--- apps/staged/src-tauri/src/store/messages.rs | 32 +- .../features/projects/projectNoteRow.test.ts | 4 +- .../sessions/sessionModalHelpers.test.ts | 29 +- .../features/sessions/sessionModalHelpers.ts | 14 +- .../timeline/sessionFailureCopy.test.ts | 12 +- .../features/timeline/sessionFailureCopy.ts | 20 +- crates/acp-client/src/driver.rs | 75 ++- crates/acp-client/src/lib.rs | 1 - 17 files changed, 1383 insertions(+), 304 deletions(-) diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index e625f721f..02cca40e3 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -53,8 +53,29 @@ pub(crate) fn auto_resume_prompt(rate_limit_kind: Option<&str>) -> &'static str } /// The bookkeeping `reason` of a network pause that waits for the gate -/// without spending a retry attempt. +/// without spending a retry attempt. The app reads it to leave the retry +/// time out of its notice (`NETWORK_WAIT_REASON` in `sessionFailureCopy.ts`): +/// such a row's `next_retry_at` is the time of the pause, not of a retry. const NETWORK_WAIT_REASON: &str = "network_wait"; +/// The bookkeeping `reason` of a network pause given up on because the +/// session's own probes kept failing with a network present. +const PROBE_LIMIT_REASON: &str = "probe_limit_exceeded"; + +/// How one pause is charged to its session's retry budget. +/// +/// Only a network pause is ever anything but `Counted`; the network gate +/// decides which (see `network_gate`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PauseAccounting { + /// Advances `attempt` under the attempt cap and the 24 hour window. + Counted, + /// Waits for the network gate: `attempt` stays, and neither the cap nor + /// the window applies. + Waiting, + /// The session's own probes failed too many times in a row with a + /// network present. It stays paused for a manual resume. + ProbesExhausted, +} const TICK_INTERVAL_SECS: u64 = 15; const MAX_AUTO_RETRY_ATTEMPTS: u32 = 8; @@ -427,7 +448,7 @@ fn ensure_auto_recovery_state( // No failure was reported for this row, so a network pause among these // is not counted: it waits for the gate like the rest. - record_rate_limit_pause_for_session(store, session, false, now)?; + record_rate_limit_pause_for_session(store, session, PauseAccounting::Waiting, now)?; Ok(store .get_session(&session.id) .map_err(|e| e.to_string())? @@ -449,11 +470,11 @@ fn ensure_auto_recovery_state( /// the blocker scan needs: `agent_id` and the message history, not status. /// /// A network pause (`kind` of [`NETWORK_PAUSE_KIND`]) is reported to the -/// scheduler's network gate, which answers whether it counts against the -/// session's retry budget: only a failure while the gate is open does. The -/// gate hears of the failure even if a Stop then wins the row, because the -/// network failed either way. With no scheduler the pause is not counted, as -/// under a fresh gate. +/// scheduler's network gate, which answers how it is charged to the +/// session's retry budget ([`PauseAccounting`]): only a failure while the +/// gate is open is counted. The gate hears of the failure even if a Stop +/// then wins the row, because the network failed either way. With no +/// scheduler the pause is not counted, as under a fresh gate. /// /// Returns whether this call won the `running → paused` transition. A lost /// race (a Stop cancelled the row first) writes nothing, including the @@ -470,12 +491,14 @@ pub(crate) fn pause_for_rate_limit( let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(false); }; - let counted = if is_network_pause(kind) { - scheduler.is_some_and(|scheduler| scheduler.record_network_failure(store, session_id, now)) + let accounting = if is_network_pause(kind) { + scheduler.map_or(PauseAccounting::Waiting, |scheduler| { + scheduler.record_network_failure(store, session_id, now) + }) } else { - true + PauseAccounting::Counted }; - let state = plan_recovery_state(store, &session, kind, counted, resets_at, now)?; + let state = plan_recovery_state(store, &session, kind, accounting, resets_at, now)?; let transitioned = store .transition_running_to_rate_limited_with_provider_metadata( session_id, @@ -496,7 +519,7 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(()); }; - record_rate_limit_pause_for_session(store, &session, true, now) + record_rate_limit_pause_for_session(store, &session, PauseAccounting::Counted, now) } /// Write retry bookkeeping onto a row that is already paused. @@ -507,14 +530,14 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< fn record_rate_limit_pause_for_session( store: &Store, session: &Session, - counted: bool, + accounting: PauseAccounting, now: i64, ) -> Result<(), String> { let state = plan_recovery_state( store, session, session.rate_limit_kind.as_deref(), - counted, + accounting, session.rate_limit_resets_at.as_deref(), now, )?; @@ -526,12 +549,12 @@ fn record_rate_limit_pause_for_session( } /// The next retry state for `session`, advancing any bookkeeping it already -/// carries. `kind` and `counted` are those of [`build_recovery_state`]. +/// carries. `kind` and `accounting` are those of [`build_recovery_state`]. fn plan_recovery_state( store: &Store, session: &Session, kind: Option<&str>, - counted: bool, + accounting: PauseAccounting, resets_at: Option<&str>, now: i64, ) -> Result { @@ -541,7 +564,7 @@ fn plan_recovery_state( Ok(build_recovery_state( existing.as_ref(), kind, - counted, + accounting, resets_at, now, blocker.as_deref(), @@ -606,9 +629,12 @@ fn log_recovery_state(session: &Session, kind: Option<&str>, state: &RateLimitAu /// so give-up is reserved for starts that keep failing rather than issued on /// the first one. /// -/// A network pause that fails to start is reported to the network gate like -/// a failed prompt, so a probe that could not start closes the gate again and -/// is counted only if the gate was open. +/// A start failure is not evidence about the network: `prepare_resume` fails +/// on a missing workspace path, a disconnected session or a branch in +/// transfer. So a network pause that fails to start is counted like any +/// other, which keeps it under the attempt cap, and if it was the network +/// gate's probe the probe is abandoned: another session may probe at once +/// and the probe delay does not grow. fn reschedule_after_start_failure( store: &Store, scheduler: &RateLimitScheduler, @@ -620,9 +646,12 @@ fn reschedule_after_start_failure( // session in any other state was claimed by something else (a manual // resume, a cancel, a delete) and is that path's to manage. Ok(Some(session)) if session.status.as_str() == "paused" => { - let counted = !is_network_pause(session.rate_limit_kind.as_deref()) - || scheduler.record_network_failure(store, session_id, now); - if let Err(e) = record_rate_limit_pause_for_session(store, &session, counted, now) { + if is_network_pause(session.rate_limit_kind.as_deref()) { + scheduler.abandon_network_probe(session_id, now); + } + if let Err(e) = + record_rate_limit_pause_for_session(store, &session, PauseAccounting::Counted, now) + { log::warn!( "[rate_limit_recovery] failed to reschedule session {session_id} after a failed start: {e}" ); @@ -650,16 +679,25 @@ fn is_actionable_for_auto_resume(session: &Session) -> bool { /// The retry state after one more pause. /// -/// A network pause (`kind` of [`NETWORK_PAUSE_KIND`]) that is not `counted` -/// failed while the network gate was not open. It keeps its `attempt`, is due -/// at once so the gate alone decides when it resumes, and is exempt from the -/// attempt cap and the 24 hour window. Every other pause, a counted network -/// one included, follows the rules below; `counted` means nothing for an -/// API-limit pause. +/// `accounting` applies to a network pause (`kind` of +/// [`NETWORK_PAUSE_KIND`]) only; an API-limit pause is always counted. +/// +/// - [`PauseAccounting::Waiting`]: the failure came while the network gate +/// was not open. The row keeps its `attempt`, is due at once so the gate +/// alone decides when it resumes, and is exempt from the attempt cap and +/// the 24 hour window. +/// - [`PauseAccounting::ProbesExhausted`]: the row gives up and waits for a +/// manual resume. +/// - [`PauseAccounting::Counted`]: the rules below, with one difference. The +/// 24 hour window of a network pause runs from its first counted failure, +/// not from its first pause: `first_limited_at` is set again when +/// `attempt` becomes 1. Time spent waiting behind a closed gate (a laptop +/// asleep for a weekend) therefore never ends a session on the first +/// failure that counts. fn build_recovery_state( existing: Option<&RateLimitAutoRecovery>, kind: Option<&str>, - counted: bool, + accounting: PauseAccounting, resets_at: Option<&str>, now: i64, blocker: Option<&str>, @@ -680,7 +718,20 @@ fn build_recovery_state( }; } - if is_network_pause(kind) && !counted { + let network = is_network_pause(kind); + if network && accounting == PauseAccounting::ProbesExhausted { + return RateLimitAutoRecovery { + attempt: existing.map(|s| s.attempt).unwrap_or(0), + max_attempts: MAX_AUTO_RETRY_ATTEMPTS, + next_retry_at: None, + first_limited_at, + updated_at: now, + gave_up_at: Some(now), + unsafe_to_auto_resume: false, + reason: Some(PROBE_LIMIT_REASON.to_string()), + }; + } + if network && accounting == PauseAccounting::Waiting { return RateLimitAutoRecovery { attempt: existing.map(|s| s.attempt).unwrap_or(0), max_attempts: MAX_AUTO_RETRY_ATTEMPTS, @@ -694,6 +745,11 @@ fn build_recovery_state( } let attempt = existing.map(|s| s.attempt.saturating_add(1)).unwrap_or(1); + let first_limited_at = if network && attempt == 1 { + now + } else { + first_limited_at + }; if attempt > MAX_AUTO_RETRY_ATTEMPTS { return RateLimitAutoRecovery { attempt, @@ -1084,7 +1140,15 @@ mod tests { let now = parse_rfc3339_millis("2026-06-24T15:00:00Z").unwrap(); let reset = "2026-06-24T15:45:00Z"; - let state = build_recovery_state(None, None, true, Some(reset), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(reset), + now, + None, + 12_000, + ); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("reset_at")); @@ -1096,7 +1160,15 @@ mod tests { #[test] fn schedules_unknown_reset_with_capped_backoff_jitter() { - let state = build_recovery_state(None, None, true, None, 1_000_000, None, 4_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + None, + 1_000_000, + None, + 4_000, + ); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("backoff")); @@ -1116,7 +1188,15 @@ mod tests { reason: Some("backoff".to_string()), }; - let state = build_recovery_state(Some(&existing), None, true, None, 10_000, None, 5_000); + let state = build_recovery_state( + Some(&existing), + None, + PauseAccounting::Counted, + None, + 10_000, + None, + 5_000, + ); assert_eq!(state.attempt, MAX_AUTO_RETRY_ATTEMPTS + 1); assert_eq!(state.gave_up_at, Some(10_000)); @@ -1129,7 +1209,7 @@ mod tests { let state = build_recovery_state( None, None, - true, + PauseAccounting::Counted, None, 10_000, Some("missing_agent_session_id"), @@ -1739,18 +1819,137 @@ mod tests { assert!(is_due(&state, 10_000)); } - /// `counted` belongs to network pauses: an API-limit pause advances its - /// attempt whatever it is given. + /// The accounting belongs to network pauses: an API-limit pause advances + /// its attempt whatever it is given. #[test] fn an_api_limit_pause_is_always_counted() { - for counted in [true, false] { - let state = - build_recovery_state(None, Some("claude_pro"), counted, None, 10_000, None, 5_000); + for accounting in [ + PauseAccounting::Counted, + PauseAccounting::Waiting, + PauseAccounting::ProbesExhausted, + ] { + let state = build_recovery_state( + None, + Some("claude_pro"), + accounting, + None, + 10_000, + None, + 5_000, + ); assert_eq!(state.attempt, 1); + assert_eq!(state.gave_up_at, None); assert_eq!(state.reason.as_deref(), Some("backoff")); } } + const DAY_MS: i64 = 24 * 60 * 60 * 1_000; + + fn network_state( + existing: Option<&RateLimitAutoRecovery>, + accounting: PauseAccounting, + now: i64, + ) -> RateLimitAutoRecovery { + build_recovery_state( + existing, + Some(NETWORK_PAUSE_KIND), + accounting, + None, + now, + None, + 5_000, + ) + } + + /// A row that waited behind the gate for more than a day, then fails + /// while the gate is open: its first counted failure starts the 24 hour + /// window, so it is scheduled, not given up on. + #[test] + fn the_first_counted_network_failure_after_a_long_wait_does_not_give_up() { + let paused_at = 1_000_000; + let waiting = network_state(None, PauseAccounting::Waiting, paused_at); + assert_eq!(waiting.first_limited_at, paused_at); + + let failed_at = paused_at + 30 * 60 * 60 * 1_000; + let counted = network_state(Some(&waiting), PauseAccounting::Counted, failed_at); + + assert_eq!(counted.attempt, 1); + assert_eq!(counted.gave_up_at, None); + assert_eq!(counted.reason.as_deref(), Some("backoff")); + assert_eq!(counted.first_limited_at, failed_at); + assert!(counted.next_retry_at.unwrap() > failed_at); + } + + /// The window still ends a network pause: 24 hours after its first + /// counted failure, the next counted one gives up. Waits in between do + /// not move the start. + #[test] + fn the_network_window_runs_from_the_first_counted_failure() { + let first_counted_at = 1_000_000; + let first = network_state(None, PauseAccounting::Counted, first_counted_at); + assert_eq!(first.first_limited_at, first_counted_at); + + let waiting = network_state( + Some(&first), + PauseAccounting::Waiting, + first_counted_at + DAY_MS / 2, + ); + assert_eq!(waiting.attempt, 1); + assert_eq!(waiting.first_limited_at, first_counted_at); + + let inside = network_state( + Some(&waiting), + PauseAccounting::Counted, + first_counted_at + DAY_MS, + ); + assert_eq!(inside.attempt, 2); + assert_eq!(inside.gave_up_at, None); + assert_eq!(inside.first_limited_at, first_counted_at); + + let outside_at = first_counted_at + DAY_MS + 1; + let outside = network_state(Some(&waiting), PauseAccounting::Counted, outside_at); + assert_eq!(outside.gave_up_at, Some(outside_at)); + assert_eq!(outside.reason.as_deref(), Some("retry_window_elapsed")); + } + + /// An API-limit pause keeps measuring from its first pause. + #[test] + fn the_api_limit_window_still_runs_from_the_first_pause() { + let first = build_recovery_state( + None, + Some("claude_pro"), + PauseAccounting::Counted, + None, + 1_000_000, + None, + 5_000, + ); + let late = 1_000_000 + DAY_MS + 1; + let state = build_recovery_state( + Some(&first), + Some("claude_pro"), + PauseAccounting::Counted, + None, + late, + None, + 5_000, + ); + assert_eq!(state.gave_up_at, Some(late)); + assert_eq!(state.reason.as_deref(), Some("retry_window_elapsed")); + } + + #[test] + fn exhausted_probes_give_up_and_wait_for_a_manual_resume() { + let waiting = network_state(None, PauseAccounting::Waiting, 10_000); + let state = network_state(Some(&waiting), PauseAccounting::ProbesExhausted, 20_000); + + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, Some(20_000)); + assert_eq!(state.next_retry_at, None); + assert_eq!(state.reason.as_deref(), Some("probe_limit_exceeded")); + assert!(!is_due(&state, i64::MAX)); + } + /// A network pause with an unanswered tool call is manual-only whether or /// not it is counted. #[test] @@ -1758,7 +1957,7 @@ mod tests { let state = build_recovery_state( None, Some(NETWORK_PAUSE_KIND), - false, + PauseAccounting::Waiting, None, 10_000, Some("pending_tool_call"), @@ -1870,18 +2069,42 @@ mod tests { // millennia out). Waiting for it would park the session forever: the // 24h give-up is only evaluated when recovery state is rebuilt. let micros = (parse_rfc3339_millis("2026-06-24T15:45:00Z").unwrap() * 1_000).to_string(); - let state = build_recovery_state(None, None, true, Some(µs), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(µs), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("backoff")); assert_eq!(jitter_bound_for(Some(µs), now), MAX_BACKOFF_MS); // The cap is on lead time, not scale: just beyond the window falls // back, just within it is honored. let beyond_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS + 60_000).to_string(); - let state = build_recovery_state(None, None, true, Some(&beyond_window), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(&beyond_window), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("backoff")); let within_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS - 60_000).to_string(); - let state = build_recovery_state(None, None, true, Some(&within_window), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(&within_window), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("reset_at")); assert_eq!(jitter_bound_for(Some(&within_window), now), RESET_JITTER_MS); } diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs index 29172d0e8..28a9fc6e6 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs @@ -6,29 +6,46 @@ //! one gate instead of each running its own backoff. API-limit pauses never //! pass through it. //! -//! | State | Meaning | What the scheduler does | -//! | ------- | ------------------------------- | -------------------------------- | -//! | Closed | The network is presumed down | Starts one probe when it is due | -//! | Probing | One probe turn is running | Starts no other network resume | -//! | Open | A probe produced agent output | Resumes the rest, two per tick | +//! | State | Meaning | What the scheduler does | +//! | ------- | -------------------------------- | ------------------------------- | +//! | Closed | The network is presumed down | Starts one probe when it is due | +//! | Probing | One probe turn is running | Starts no other network resume | +//! | Open | A probe reached its provider | Resumes the rest, two per tick | //! //! The gate lives in memory, owned by [`RateLimitScheduler`] behind a mutex, //! and starts closed with a probe due at once, so a relaunch probes on its //! first tick. //! +//! A probe succeeds when its turn reaches the model provider (see +//! [`probe_verdict`]). A probe still running after [`PROBE_TIMEOUT_MS`] with +//! no such sign is given up on as a probe: the gate closes and the schedule +//! moves on as after a failed probe, while that session keeps running. A +//! user's manual resume of a network-paused session is watched as a probe +//! too when the gate is closed, so its success opens the gate for the rest. +//! //! Failures while the gate is not open are not counted: the session's //! `attempt` does not advance and the 24 hour window is not applied, so a //! laptop that sleeps for days gives up on nothing. Failures while it is open //! are counted under the existing cap, and close it again. //! +//! One bound applies to uncounted failures. A session whose own probes fail +//! [`MAX_CONSECUTIVE_PROBE_FAILURES`] times in a row, each while the machine +//! stayed awake and held a network address, is given up on and left paused +//! for a manual resume: with a network evidently present, its failure is not +//! one that waiting fixes (a dead local proxy, a captive portal). +//! //! Two things make a closed gate probe sooner than its doubling delay, both //! derived inside the scheduler tick with no operating-system callbacks: //! //! - **Wake.** A gap of more than [`WAKE_GAP_MS`] between two ticks means the //! process was suspended. The probe delay resets, so a probe is due at once. //! - **Network change.** While the gate is closed, each tick reads the -//! machine's [`NetworkFingerprint`]. A change resets the probe delay. An -//! empty set means no network, and no probe starts. +//! machine's [`NetworkFingerprint`]. A change resets the probe delay. +//! +//! An empty fingerprint means the machine has no network, so neither trigger +//! fires while it is empty. It never blocks a probe: the doubling schedule +//! keeps running, because the provider may be local (a model server or a +//! gateway on this machine) and reachable with no network at all. //! //! A probe starts only on a tick whose preceding gap was normal. A short //! maintenance wake during sleep cannot produce two ticks one interval apart, @@ -36,12 +53,12 @@ //! //! Every method takes `now`; the gate never reads a clock of its own. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::net::IpAddr; -use crate::store::{SessionStatus, Store}; +use crate::store::{CompletionReason, Session, SessionStatus, Store}; -use super::RateLimitScheduler; +use super::{is_network_pause, PauseAccounting, RateLimitScheduler}; mod fingerprint; @@ -54,6 +71,14 @@ pub(super) const MAX_PROBE_DELAY_MS: i64 = 10 * 60_000; /// A gap between two ticks longer than this means the process was suspended: /// three of the scheduler's 15 second ticks. pub(super) const WAKE_GAP_MS: i64 = 45_000; +/// How long a probe may run without reaching its provider before the gate +/// stops waiting for it. Longer than a provider request takes to time out, +/// so a probe that will fail has normally failed by then. +pub(super) const PROBE_TIMEOUT_MS: i64 = 10 * 60_000; +/// How many times in a row one session's own probes may fail, each while the +/// machine stayed awake and held a network address, before the session is +/// given up on and left paused for a manual resume. +pub(super) const MAX_CONSECUTIVE_PROBE_FAILURES: u32 = 20; #[derive(Debug, Clone, PartialEq, Eq)] enum GateState { @@ -62,6 +87,9 @@ enum GateState { Probing { session_id: String, since: i64, + /// The process was suspended while the probe ran, so its failure + /// says nothing about a network the machine was awake to use. + slept: bool, }, Open, } @@ -92,6 +120,9 @@ pub(super) struct NetworkGate { /// The fingerprint last read while the gate was closed. `None` until the /// first read, after the gate opens, and when a read fails. addresses: Option>, + /// Per session, how many of its own probes failed in a row while the + /// machine was awake with a network address. Cleared when the gate opens. + probe_failures: HashMap, } impl Default for NetworkGate { @@ -103,6 +134,7 @@ impl Default for NetworkGate { last_tick_at: None, settled: true, addresses: None, + probe_failures: HashMap::new(), } } } @@ -112,17 +144,14 @@ impl NetworkGate { match self.state { GateState::Open => NetworkTurn::Resume, GateState::Probing { .. } => NetworkTurn::Hold, - GateState::Closed - if now >= self.next_probe_at && self.settled && !self.network_absent() => - { - NetworkTurn::Probe - } + GateState::Closed if now >= self.next_probe_at && self.settled => NetworkTurn::Probe, GateState::Closed => NetworkTurn::Hold, } } /// Record a scheduler tick, and answer whether the gap before it shows - /// that the process was suspended. + /// that the process was suspended. The caller decides whether that + /// resets the probe delay (see [`Self::apply_triggers`]). fn observe_tick(&mut self, now: i64) -> bool { let woke = self .last_tick_at @@ -130,7 +159,9 @@ impl NetworkGate { self.last_tick_at = Some(now); self.settled = !woke; if woke { - self.reset_probe_delay(now); + if let GateState::Probing { slept, .. } = &mut self.state { + *slept = true; + } } woke } @@ -145,6 +176,18 @@ impl NetworkGate { } } + /// Act on this tick's triggers: a wake, or a changed fingerprint. Either + /// resets the probe delay, unless the fingerprint is empty: a machine + /// with no network has nothing new to probe, so the doubling schedule + /// stands. Answers whether the delay was reset. + fn apply_triggers(&mut self, woke: bool, changed: bool, now: i64) -> bool { + let fire = (woke || changed) && !self.network_absent(); + if fire { + self.reset_probe_delay(now); + } + fire + } + fn is_closed(&self) -> bool { self.state == GateState::Closed } @@ -154,10 +197,18 @@ impl NetworkGate { self.addresses.as_ref().is_some_and(BTreeSet::is_empty) } + /// The last fingerprint read held an address. `false` when it held none + /// and when nothing is known. + fn network_present(&self) -> bool { + self.addresses + .as_ref() + .is_some_and(|addresses| !addresses.is_empty()) + } + /// Record the fingerprint read on this tick, and answer whether it /// differs from the one before. The first read is the baseline, not a /// change. - fn observe_fingerprint(&mut self, addresses: BTreeSet, now: i64) -> bool { + fn observe_fingerprint(&mut self, addresses: BTreeSet) -> bool { if !self.is_closed() { return false; } @@ -166,14 +217,12 @@ impl NetworkGate { .as_ref() .is_some_and(|previous| *previous != addresses); self.addresses = Some(addresses); - if changed { - self.reset_probe_delay(now); - } changed } /// The fingerprint could not be read. Nothing is known about the - /// network, so an earlier "no network" answer no longer blocks a probe. + /// network, so an earlier "no network" answer no longer holds back the + /// triggers. fn fingerprint_unreadable(&mut self) { self.addresses = None; } @@ -182,12 +231,26 @@ impl NetworkGate { self.state = GateState::Probing { session_id: session_id.to_string(), since: now, + slept: false, }; } + /// The user resumed a network-paused session by hand. A closed gate + /// watches that turn as its probe, so its answer opens the gate for the + /// rest; a gate already probing or open has its answer coming. + fn adopt_probe(&mut self, session_id: &str, now: i64) -> bool { + let adopt = self.is_closed(); + if adopt { + self.probe_started(session_id, now); + } + adopt + } + fn probe(&self) -> Option<(String, i64)> { match &self.state { - GateState::Probing { session_id, since } => Some((session_id.clone(), *since)), + GateState::Probing { + session_id, since, .. + } => Some((session_id.clone(), *since)), _ => None, } } @@ -196,11 +259,12 @@ impl NetworkGate { matches!(&self.state, GateState::Probing { session_id: probe, .. } if probe == session_id) } - /// The probe's turn produced agent output: the network is back. + /// The probe's turn reached its provider: the network is back. fn probe_succeeded(&mut self, session_id: &str) { if self.is_probing(session_id) { self.state = GateState::Open; self.probe_delay_ms = BASE_PROBE_DELAY_MS; + self.probe_failures.clear(); // The fingerprint is not read while the gate is open, so the // next closed period starts from a fresh baseline. self.addresses = None; @@ -217,33 +281,70 @@ impl NetworkGate { } } - /// Record that `session_id` failed on the network, and answer whether the - /// failure counts against that session's retry budget. - fn record_failure(&mut self, session_id: &str, now: i64) -> bool { + /// The probe has run for [`PROBE_TIMEOUT_MS`] without reaching its + /// provider. The gate stops waiting for it: the schedule moves on as + /// after a failed probe, and the session, still running, is left alone. + fn probe_timed_out(&mut self, session_id: &str, now: i64) { + if self.is_probing(session_id) { + self.close_after_failed_probe(now); + } + } + + /// Close the gate after a probe that did not get through: the next one + /// waits the current delay, and the one after it twice that. + fn close_after_failed_probe(&mut self, now: i64) { + self.state = GateState::Closed; + self.next_probe_at = now.saturating_add(self.probe_delay_ms); + self.probe_delay_ms = self + .probe_delay_ms + .saturating_mul(2) + .min(MAX_PROBE_DELAY_MS); + } + + /// Record that `session_id` failed on the network, and answer how the + /// failure is charged to that session's retry budget. + fn record_failure(&mut self, session_id: &str, now: i64) -> PauseAccounting { match &self.state { GateState::Open => { self.state = GateState::Closed; self.next_probe_at = now.saturating_add(BASE_PROBE_DELAY_MS); self.probe_delay_ms = BASE_PROBE_DELAY_MS.saturating_mul(2); - true + PauseAccounting::Counted } GateState::Probing { - session_id: probe, .. + session_id: probe, + slept, + .. } if probe == session_id => { - self.state = GateState::Closed; - self.next_probe_at = now.saturating_add(self.probe_delay_ms); - self.probe_delay_ms = self - .probe_delay_ms - .saturating_mul(2) - .min(MAX_PROBE_DELAY_MS); - false + let awake = !*slept; + self.close_after_failed_probe(now); + if awake && self.network_present() && self.probe_failures_exhausted(session_id) { + PauseAccounting::ProbesExhausted + } else { + PauseAccounting::Waiting + } } // A session that was already running when the network dropped, // failing while the gate is closed or another session probes. It // says nothing new, so the probe schedule stands. - GateState::Probing { .. } | GateState::Closed => false, + GateState::Probing { .. } | GateState::Closed => PauseAccounting::Waiting, } } + + /// Count one more failed probe of `session_id` made with the machine + /// awake and a network present, and answer whether that was its last. + fn probe_failures_exhausted(&mut self, session_id: &str) -> bool { + let failures = self + .probe_failures + .entry(session_id.to_string()) + .or_insert(0); + *failures += 1; + let exhausted = *failures >= MAX_CONSECUTIVE_PROBE_FAILURES; + if exhausted { + self.probe_failures.remove(session_id); + } + exhausted + } } /// How a running probe stands, read from the store. @@ -254,12 +355,16 @@ enum ProbeVerdict { Abandoned, } -/// A probe succeeds when its turn records any agent message, tool call or -/// prompt response after the claim. A turn that completed did get its prompt -/// response, whether or not the row for it is in the store yet. +/// A probe succeeds when its turn shows, after the claim, that a request +/// reached the provider: a tool call or its result, a result the agent did +/// not mark as an error, or a prompt response (see +/// [`Store::session_reached_provider_since`]). Message text does not count: +/// a failed turn records its own error as text. A turn that completed did +/// get its prompt response, whether or not the row for it is in the store +/// yet. fn probe_verdict(store: &Store, session_id: &str, since: i64) -> Result { if store - .session_has_agent_output_since(session_id, since) + .session_reached_provider_since(session_id, since) .map_err(|e| e.to_string())? { return Ok(ProbeVerdict::Succeeded); @@ -290,34 +395,44 @@ impl RateLimitScheduler { /// read only while the gate is closed and `network_rows_waiting`, so a /// machine with nothing paused on the network never lists its interfaces. pub(super) fn observe_network_tick(&self, now: i64, network_rows_waiting: bool) { - let closed = { + let woke = { let mut gate = self.network_gate.lock().unwrap(); - if gate.observe_tick(now) { - log::info!("[network_gate] woke from a suspend; probe delay reset"); + let woke = gate.observe_tick(now); + if !gate.is_closed() || !network_rows_waiting { + // No fingerprint is read, so nothing says the network is + // absent: a wake resets the delay. + if woke && gate.apply_triggers(true, false, now) { + log::info!("[network_gate] woke from a suspend; probe delay reset"); + } + return; } - gate.is_closed() + woke }; - if !closed || !network_rows_waiting { - return; - } // The interfaces are read with the gate unlocked; the gate rechecks // that it is still closed before it takes the answer. let addresses = self.network_fingerprint.read(); let mut gate = self.network_gate.lock().unwrap(); - match addresses { - Ok(addresses) => { - let was_absent = gate.network_absent(); - let changed = gate.observe_fingerprint(addresses, now); - if gate.network_absent() && !was_absent { - log::info!("[network_gate] no network interface has an address; not probing"); - } else if changed { - log::info!("[network_gate] network interfaces changed; probe delay reset"); - } - } + let changed = match addresses { + Ok(addresses) => gate.observe_fingerprint(addresses), Err(e) => { log::warn!("[network_gate] failed to read network interfaces: {e}"); gate.fingerprint_unreadable(); + false } + }; + if gate.apply_triggers(woke, changed, now) { + log::info!( + "[network_gate] {}; probe delay reset", + if woke { + "woke from a suspend" + } else { + "network interfaces changed" + } + ); + } else if woke || changed { + log::info!( + "[network_gate] no network interface has an address; probe schedule unchanged" + ); } } @@ -330,19 +445,52 @@ impl RateLimitScheduler { .probe_started(session_id, now); } - /// Move the gate out of `Probing` if the store shows how the probe went. + /// The user resumed `session` by hand and the claim won. If it was paused + /// on the network and the gate is closed, the gate watches the turn as + /// its probe: without this a resume that gets through tells the gate + /// nothing, and the other paused sessions wait out the probe delay. + pub(crate) fn manual_resume_started(&self, session: &Session, now: i64) { + let network_paused = session.status == SessionStatus::Paused + && session.completion_reason == Some(CompletionReason::RateLimited) + && is_network_pause(session.rate_limit_kind.as_deref()); + if network_paused + && self + .network_gate + .lock() + .unwrap() + .adopt_probe(&session.id, now) + { + log::info!( + "[network_gate] watching manually resumed session {} as the probe", + session.id + ); + } + } + + /// The probe `session_id` ended without saying anything about the + /// network: its resume failed to start. Another session may probe at + /// once, and the probe delay is left alone. + pub(super) fn abandon_network_probe(&self, session_id: &str, now: i64) { + self.network_gate + .lock() + .unwrap() + .probe_abandoned(session_id, now); + } + + /// Move the gate out of `Probing` if the store shows how the probe went, + /// or if it has run past [`PROBE_TIMEOUT_MS`]. /// /// Runs at the start of each tick, and before a network failure is - /// recorded so that a probe which produced output and then failed is - /// counted as a failure with the gate open. + /// recorded so that a probe which reached its provider and then failed + /// is counted as a failure with the gate open. pub(super) fn settle_network_probe(&self, store: &Store, now: i64) { self.settle_probe(store, None, now); } /// [`Self::settle_network_probe`], with the session whose network failure - /// is about to be recorded. If that session is the probe, a row that no - /// longer runs is its failure, not an abandoned probe: a probe that fails - /// to start is parked again before the gate hears of it. + /// is about to be recorded. If that session is the probe, its failure is + /// what settles it: a row that no longer runs is not an abandoned probe, + /// and one that ran long is not timed out. fn settle_probe(&self, store: &Store, failing: Option<&str>, now: i64) { // The store is read with the gate unlocked; the gate rechecks that it // is still probing this session before it moves. @@ -350,6 +498,19 @@ impl RateLimitScheduler { return; }; match probe_verdict(store, &session_id, since) { + Ok(ProbeVerdict::Running | ProbeVerdict::Abandoned) + if failing == Some(session_id.as_str()) => {} + Ok(ProbeVerdict::Running) if now.saturating_sub(since) >= PROBE_TIMEOUT_MS => { + log::warn!( + "[network_gate] probe session {session_id} has not reached its provider \ + after {} s; no longer waiting for it", + PROBE_TIMEOUT_MS / 1_000 + ); + self.network_gate + .lock() + .unwrap() + .probe_timed_out(&session_id, now); + } Ok(ProbeVerdict::Running) => {} Ok(ProbeVerdict::Succeeded) => { log::info!("[network_gate] probe session {session_id} got through; gate open"); @@ -358,7 +519,6 @@ impl RateLimitScheduler { .unwrap() .probe_succeeded(&session_id); } - Ok(ProbeVerdict::Abandoned) if failing == Some(session_id.as_str()) => {} Ok(ProbeVerdict::Abandoned) => { log::info!("[network_gate] probe session {session_id} ended without an answer"); self.network_gate @@ -372,21 +532,26 @@ impl RateLimitScheduler { } } - /// Record that `session_id` failed on the network, and answer whether the - /// failure counts against that session's retry budget: it does only when - /// the gate was open. - pub(super) fn record_network_failure(&self, store: &Store, session_id: &str, now: i64) -> bool { + /// Record that `session_id` failed on the network, and answer how the + /// failure is charged to that session's retry budget: counted only when + /// the gate was open, and the end of the session's automatic resumes when + /// its own probes have failed [`MAX_CONSECUTIVE_PROBE_FAILURES`] times. + pub(super) fn record_network_failure( + &self, + store: &Store, + session_id: &str, + now: i64, + ) -> PauseAccounting { self.settle_probe(store, Some(session_id), now); - let counted = self + let accounting = self .network_gate .lock() .unwrap() .record_failure(session_id, now); log::info!( - "[network_gate] session {session_id} failed on the network ({}); gate closed", - if counted { "counted" } else { "not counted" } + "[network_gate] session {session_id} failed on the network ({accounting:?}); gate closed" ); - counted + accounting } } diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs index cb986515b..554f5fc50 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs @@ -1,6 +1,8 @@ //! The machine's network fingerprint: the set of addresses its interfaces //! hold. The gate reads it while closed to notice that the network changed, -//! or that there is none. +//! or that there is none. "None" only holds back the wake and change +//! triggers; it never stops a probe, since a provider on this machine or on +//! a link-local network needs none of these addresses. use std::collections::BTreeSet; use std::io; diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs index 8b5b9e47b..655226d08 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs @@ -4,13 +4,16 @@ use std::time::Duration; use super::super::{ admit_for_resume, auto_recovery_from_metadata, claim_for_auto_resume, network_rows_waiting, - pause_for_rate_limit, reschedule_after_start_failure, AutoResumeClaim, RateLimitAutoRecovery, - RateLimitScheduler, MAX_RESUMES_PER_TICK, NETWORK_PAUSE_KIND, + pause_for_rate_limit, reschedule_after_start_failure, AutoResumeClaim, PauseAccounting, + RateLimitAutoRecovery, RateLimitScheduler, MAX_AUTO_RETRY_ATTEMPTS, MAX_RESUMES_PER_TICK, + NETWORK_PAUSE_KIND, }; use super::*; mod triggers; -use crate::store::{CompletionReason, MessageRole, Session, SessionStatus, Store}; +use crate::store::{ + AcpMessageMetadata, CompletionReason, MessageRole, Session, SessionStatus, Store, +}; const HOUR_MS: i64 = 60 * 60 * 1_000; @@ -99,9 +102,84 @@ fn recovery(store: &Store, session_id: &str) -> RateLimitAutoRecovery { auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap() } +/// The agent's turn reaches its provider: the model asks for a tool, and +/// the tool answers. fn agent_replies(store: &Store, session_id: &str) { + for (role, content) in [ + (MessageRole::ToolCall, "Read"), + (MessageRole::ToolResult, "contents"), + ] { + store + .add_session_message(session_id, role, content) + .unwrap(); + } +} + +const OFFLINE_ERROR: &str = + "API Error: Can't reach the API server \u{2014} check your internet or DNS (ENOTFOUND)"; + +fn acp_event(store: &Store, session_id: &str, kind: &str, content: serde_json::Value) { store - .add_session_message(session_id, MessageRole::Assistant, "Back online.") + .add_acp_metadata_message( + session_id, + &AcpMessageMetadata { + acp_event_kind: Some(kind.to_string()), + acp_content: Some(content), + ..Default::default() + }, + ) + .unwrap(); +} + +/// The rows a Claude turn records when it cannot reach the provider, as +/// stored by real failed turns: the CLI's error as a message chunk, the +/// SDK's result marked as an error (its `subtype` still reads `success`), a +/// usage update, the prompt error, and the error text as the visible +/// assistant message. +fn claude_turn_fails_offline(store: &Store, session_id: &str) { + acp_event( + store, + session_id, + "agent_message_chunk", + serde_json::json!({ + "content": { "type": "text", "text": OFFLINE_ERROR }, + "messageId": "a6922e42-33a1-4c9b-95d6-45c748f7e168", + }), + ); + acp_event( + store, + session_id, + "response_result", + serde_json::json!({ + "type": "result", + "subtype": "success", + "is_error": true, + "terminal_reason": "api_error", + "api_error": "connection_failed", + "api_error_status": null, + "stop_reason": "stop_sequence", + "num_turns": 32, + "result": OFFLINE_ERROR, + }), + ); + acp_event( + store, + session_id, + "usage_update", + serde_json::json!({ "used": 175_506, "size": 1_000_000 }), + ); + acp_event( + store, + session_id, + "prompt_error", + serde_json::json!({ + "code": -32603, + "message": format!("Internal error: {OFFLINE_ERROR}"), + "data": { "errorKind": "server_error" }, + }), + ); + store + .add_session_message(session_id, MessageRole::Assistant, OFFLINE_ERROR) .unwrap(); } @@ -137,7 +215,7 @@ fn only_one_probe_starts_for_eleven_paused_rows() { assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); // While the probe runs, later ticks start nothing. assert!(tick(&scheduler, &store, now + 15_000).is_empty()); - assert!(tick(&scheduler, &store, now + HOUR_MS).is_empty()); + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS - 1).is_empty()); assert_eq!(store.get_rate_limited_paused_sessions().unwrap().len(), 10); } @@ -185,7 +263,10 @@ fn a_probe_with_agent_output_opens_the_gate_and_the_rest_resume_two_per_tick() { fail_on_network(&store, &scheduler, &first.id, now); let rest = network_paused_sessions(&store, &scheduler, 4, now); - assert_eq!(tick(&scheduler, &store, now), [first.id.clone()]); + assert_eq!( + tick(&scheduler, &store, now), + std::slice::from_ref(&first.id) + ); assert!(tick(&scheduler, &store, now).is_empty()); agent_replies(&store, &first.id); @@ -336,7 +417,10 @@ fn api_limit_rows_are_unaffected_by_the_gate() { // It resumes on its own schedule, past the network row the gate holds. let due = state.next_retry_at.unwrap(); assert!(tick(&scheduler, &store, due - 1).is_empty()); - assert_eq!(tick(&scheduler, &store, due), [limited.id.clone()]); + assert_eq!( + tick(&scheduler, &store, due), + std::slice::from_ref(&limited.id) + ); assert_eq!(scheduler.network_turn(due), NetworkTurn::Hold); // Nor does an API limit close an open gate. @@ -356,33 +440,298 @@ fn api_limit_rows_are_unaffected_by_the_gate() { assert_eq!(scheduler.network_turn(due), NetworkTurn::Resume); } +/// The failed start re-parks the row, then reschedules it. +fn fail_to_start(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + assert!(store + .transition_running_to_rate_limited( + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + ) + .unwrap()); + reschedule_after_start_failure(store, scheduler, session_id, now); +} + +/// A start failure says nothing about the network: the probe is abandoned, +/// so the delay does not grow and another row probes at once, and the +/// failure is counted against the session that could not start. #[test] -fn a_probe_that_fails_to_start_closes_the_gate_uncounted() { +fn a_probe_that_fails_to_start_is_counted_and_hands_the_probe_over() { let store = Arc::new(Store::in_memory().unwrap()); let scheduler = RateLimitScheduler::new(); let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + fail_to_start(&store, &scheduler, &sessions[0], now); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert!(state.next_retry_at.unwrap() > now); + + // The next row probes on the same tick time, with no delay added. + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + // And when that probe fails on the network, the delay is still the + // base one: the failed start doubled nothing. + fail_on_network(&store, &scheduler, &sessions[1], now); + let due = state.next_retry_at.unwrap().max(now + BASE_PROBE_DELAY_MS); + assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert!(!tick(&scheduler, &store, due).is_empty()); +} + +/// A lone network-paused session whose resume never starts is bounded by +/// the attempt cap, as it was before the gate existed. +#[test] +fn a_lone_session_that_never_starts_gives_up_at_the_attempt_cap() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let mut now = crate::store::now_timestamp(); let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + for attempt in 1..=MAX_AUTO_RETRY_ATTEMPTS { + assert_eq!(tick(&scheduler, &store, now), sessions, "attempt {attempt}"); + fail_to_start(&store, &scheduler, &sessions[0], now); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, attempt); + assert_eq!(state.gave_up_at, None); + now = state.next_retry_at.unwrap(); + } + assert_eq!(tick(&scheduler, &store, now), sessions); + fail_to_start(&store, &scheduler, &sessions[0], now); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.gave_up_at, Some(now)); + assert_eq!(state.reason.as_deref(), Some("retry_cap_exceeded")); + assert!(tick(&scheduler, &store, now + HOUR_MS).is_empty()); +} - // The failed start re-parks the row, then reschedules it. - assert!(store - .transition_running_to_rate_limited( +/// What a real failed Claude turn records is not a sign of the network: the +/// error text, the error result and the visible message all follow the +/// claim, and none of them may open the gate or get the failure counted. +#[test] +fn a_failed_claude_probe_is_not_taken_for_a_success() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 3, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + claude_turn_fails_offline(&store, &sessions[0]); + // A tick that lands between those rows and the pause write resumes + // nobody into the outage. + assert!(tick(&scheduler, &store, now).is_empty()); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); + + let failed_at = now + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + + // The gate closed as after any failed probe, and the delay doubles. + assert!(tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + let second_at = failed_at + BASE_PROBE_DELAY_MS; + assert_eq!(tick(&scheduler, &store, second_at), [sessions[1].clone()]); + claude_turn_fails_offline(&store, &sessions[1]); + fail_on_network(&store, &scheduler, &sessions[1], second_at); + assert_eq!(recovery(&store, &sessions[1]).attempt, 0); + assert!(tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS), + [sessions[2].clone()] + ); +} + +/// The signs that do open the gate: a result the agent did not mark as an +/// error, and a prompt response. Message text alone does not. +#[test] +fn a_successful_result_or_prompt_response_opens_the_gate_and_text_does_not() { + for kind in ["response_result", "prompt_response"] { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + store + .add_session_message(&sessions[0], MessageRole::Assistant, "Checking my work.") + .unwrap(); + acp_event( + &store, &sessions[0], - Some("Can't reach the API server"), - None, - Some(NETWORK_PAUSE_KIND), - ) - .unwrap()); - reschedule_after_start_failure(&store, &scheduler, &sessions[0], now); + "agent_message_chunk", + serde_json::json!({ "content": { "type": "text", "text": "Checking my work." } }), + ); + assert!(tick(&scheduler, &store, now).is_empty(), "{kind}"); + + let content = if kind == "response_result" { + serde_json::json!({ "type": "result", "subtype": "success", "is_error": false }) + } else { + serde_json::json!({ "stopReason": "end_turn" }) + }; + acp_event(&store, &sessions[0], kind, content); + assert_eq!( + tick(&scheduler, &store, now), + [sessions[1].clone()], + "{kind}" + ); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); + } +} + +/// After an outage longer than 24 hours, a session's first counted failure +/// schedules a retry: the window starts there, not at the first pause. +#[test] +fn a_counted_failure_after_a_long_outage_does_not_give_up() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + // The ticks run in the past so that the probe's reply, stamped with + // the wall clock, lands after the probe's claim. + let paused_at = crate::store::now_timestamp() - 40 * HOUR_MS; + let sessions = network_paused_sessions(&store, &scheduler, 1, paused_at); + + let resumed_at = paused_at + 30 * HOUR_MS; + assert_eq!(tick(&scheduler, &store, resumed_at), sessions); + // The probe gets through, then the connection drops again. + agent_replies(&store, &sessions[0]); + let failed_at = resumed_at + 60_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 1); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert_eq!(state.first_limited_at, failed_at); +} + +/// A probe that stays `running` with no sign of the provider stops holding +/// the other rows after the timeout. +#[test] +fn a_probe_that_hangs_times_out_and_another_row_probes() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS - 1).is_empty()); + assert_eq!( + scheduler.network_gate.lock().unwrap().probe(), + Some((sessions[0].clone(), now)) + ); + + // The timeout closes the gate like a failed probe: the next one waits + // the probe delay. + let timed_out_at = now + PROBE_TIMEOUT_MS; + assert!(tick(&scheduler, &store, timed_out_at).is_empty()); + assert_eq!(scheduler.network_gate.lock().unwrap().probe(), None); + assert!(tick(&scheduler, &store, timed_out_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, timed_out_at + BASE_PROBE_DELAY_MS), + [sessions[1].clone()] + ); + + // The session that hung is still running. When it fails at last, it is + // not counted and does not disturb the probe now running. + let late = timed_out_at + BASE_PROBE_DELAY_MS + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], late); assert_eq!(recovery(&store, &sessions[0]).attempt, 0); - assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert!(scheduler + .network_gate + .lock() + .unwrap() + .is_probing(&sessions[1])); +} + +/// The user's "Resume now" on a network-paused row: the claim the manual +/// resume makes, with the word it gives the scheduler. +fn resume_manually(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + let paused = store.get_session(session_id).unwrap().unwrap(); + assert!(store + .transition_to_running_clearing_agent_id(session_id, None) + .unwrap()); + scheduler.manual_resume_started(&paused, now); +} + +#[test] +fn a_manual_resume_that_gets_through_opens_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 4, now); + // A failed probe leaves the gate closed with the next probe a delay away. + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + fail_on_network(&store, &scheduler, &sessions[0], now); + assert!(tick(&scheduler, &store, now).is_empty()); + + resume_manually(&store, &scheduler, &sessions[1], now); + // The gate watches that turn; nothing else starts meanwhile, even once + // the probe delay is up. + assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS).is_empty()); + + agent_replies(&store, &sessions[1]); + assert_eq!(tick(&scheduler, &store, now + 1_000).len(), 2); + assert_eq!(scheduler.network_turn(now + 1_000), NetworkTurn::Resume); +} + +#[test] +fn a_manual_resume_that_fails_closes_the_gate_uncounted() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + fail_on_network(&store, &scheduler, &sessions[0], now); + + resume_manually(&store, &scheduler, &sessions[1], now); + claude_turn_fails_offline(&store, &sessions[1]); + fail_on_network(&store, &scheduler, &sessions[1], now); + + assert_eq!(recovery(&store, &sessions[1]).attempt, 0); + // It failed as the second probe in a row: the next waits twice the base. + assert!(tick(&scheduler, &store, now + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); assert_eq!( - tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS), - sessions + tick(&scheduler, &store, now + 2 * BASE_PROBE_DELAY_MS).len(), + 1 ); } +/// Only a network pause is watched, and only by a closed gate. +#[test] +fn a_manual_resume_of_another_row_tells_the_gate_nothing() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + + let limited = running_session(&store); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + now, + ) + .unwrap()); + resume_manually(&store, &scheduler, &limited.id, now); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Probe); + + // A probe is already running: it keeps the gate. + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + resume_manually(&store, &scheduler, &sessions[1], now); + assert!(scheduler + .network_gate + .lock() + .unwrap() + .is_probing(&sessions[0])); +} + #[test] fn the_probe_delay_doubles_to_a_ten_minute_cap() { let mut gate = NetworkGate::default(); @@ -391,7 +740,7 @@ fn the_probe_delay_doubles_to_a_ten_minute_cap() { for _ in 0..8 { assert_eq!(gate.turn(now), NetworkTurn::Probe); gate.probe_started("probe", now); - assert!(!gate.record_failure("probe", now)); + assert_eq!(gate.record_failure("probe", now), PauseAccounting::Waiting); delays.push(gate.next_probe_at - now); now = gate.next_probe_at; } @@ -403,11 +752,11 @@ fn the_probe_delay_doubles_to_a_ten_minute_cap() { // An open period starts the doubling again. gate.probe_started("probe", now); gate.probe_succeeded("probe"); - assert!(gate.record_failure("other", now)); + assert_eq!(gate.record_failure("other", now), PauseAccounting::Counted); assert_eq!(gate.next_probe_at - now, BASE_PROBE_DELAY_MS); now = gate.next_probe_at; gate.probe_started("probe", now); - assert!(!gate.record_failure("probe", now)); + assert_eq!(gate.record_failure("probe", now), PauseAccounting::Waiting); assert_eq!(gate.next_probe_at - now, 2 * BASE_PROBE_DELAY_MS); } @@ -416,7 +765,10 @@ fn another_sessions_failure_leaves_a_running_probe_alone() { let mut gate = NetworkGate::default(); gate.probe_started("probe", 1_000); - assert!(!gate.record_failure("other", 2_000)); + assert_eq!( + gate.record_failure("other", 2_000), + PauseAccounting::Waiting + ); assert!(gate.is_probing("probe")); // Nor does a stale answer about a session that is not the probe move it. @@ -424,3 +776,112 @@ fn another_sessions_failure_leaves_a_running_probe_alone() { gate.probe_abandoned("other", 2_000); assert!(gate.is_probing("probe")); } + +/// A closed gate that has read a fingerprint with one address. +fn gate_with_a_network() -> NetworkGate { + let mut gate = NetworkGate::default(); + assert!(!gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()]))); + gate +} + +fn fail_a_probe(gate: &mut NetworkGate, session_id: &str, now: i64) -> PauseAccounting { + gate.probe_started(session_id, now); + gate.record_failure(session_id, now) +} + +#[test] +fn a_sessions_twentieth_failed_probe_in_a_row_exhausts_it() { + let mut gate = gate_with_a_network(); + for failure in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting, + "failure {failure}" + ); + // Another session's failed probes are its own. + assert_eq!( + fail_a_probe(&mut gate, "other", 1_000), + PauseAccounting::Waiting + ); + } + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::ProbesExhausted + ); + // The count starts over for a session that was given up on. + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting + ); +} + +#[test] +fn a_probe_failure_counts_only_awake_with_a_network_present() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + + // The machine slept while the probe ran. + gate.observe_tick(1_000); + gate.probe_started("lone", 1_000); + assert!(gate.observe_tick(1_000 + WAKE_GAP_MS + 1)); + assert_eq!( + gate.record_failure("lone", 2_000_000), + PauseAccounting::Waiting + ); + + // No interface holds an address. + assert!(gate.observe_fingerprint(BTreeSet::new())); + assert_eq!( + fail_a_probe(&mut gate, "lone", 3_000_000), + PauseAccounting::Waiting + ); + + // Nothing is known about the network. + gate.fingerprint_unreadable(); + assert_eq!( + fail_a_probe(&mut gate, "lone", 4_000_000), + PauseAccounting::Waiting + ); + + // None of those broke the run or added to it: the next failure awake + // with a network is the twentieth. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe(&mut gate, "lone", 5_000_000), + PauseAccounting::ProbesExhausted + ); +} + +#[test] +fn a_probe_that_gets_through_clears_the_failure_counts() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + gate.probe_started("other", 1_000); + gate.probe_succeeded("other"); + assert_eq!(gate.record_failure("lone", 1_000), PauseAccounting::Counted); + + // Closed again: the fingerprint is read anew, and "lone" starts at one. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting + ); +} + +#[test] +fn a_timed_out_probe_closes_the_gate_like_a_failed_one_but_is_not_counted() { + let mut gate = gate_with_a_network(); + gate.probe_started("probe", 1_000); + gate.probe_timed_out("other", 2_000); + assert!(gate.is_probing("probe")); + + gate.probe_timed_out("probe", 2_000); + assert!(gate.is_closed()); + assert_eq!(gate.next_probe_at, 2_000 + BASE_PROBE_DELAY_MS); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + assert!(gate.probe_failures.is_empty()); +} diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs index cab3003fb..8a8087efb 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs @@ -193,72 +193,184 @@ fn a_fingerprint_change_resets_the_delay() { ); } +/// Tick every 15 seconds from `from` until a probe starts, and return it +/// with the time it started. +fn next_probe(scheduler: &RateLimitScheduler, store: &Arc, from: i64) -> (String, i64) { + let mut at = from; + loop { + let mut started = tick_with_triggers(scheduler, store, at); + if let Some(session_id) = started.pop() { + assert!(started.is_empty()); + return (session_id, at); + } + at += TICK_MS; + assert!(at < from + 2 * MAX_PROBE_DELAY_MS, "no probe started"); + } +} + +/// No address at all is not a reason to stop probing: the provider may be +/// on this machine. The doubling schedule runs as it does with a network. #[test] -fn an_empty_fingerprint_blocks_the_probe_without_advancing_the_delay() { +fn an_empty_fingerprint_does_not_stop_the_doubling_probe_schedule() { let store = Arc::new(Store::in_memory().unwrap()); - let network = FakeNetwork::with_addresses(&[]); - let scheduler = network.scheduler(); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); let now = crate::store::now_timestamp(); let sessions = network_paused_sessions(&store, &scheduler, 1, now); let mut at = now; - for _ in 0..40 { - assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); - at += TICK_MS; + let mut waits = Vec::new(); + for _ in 0..4 { + let (probe, started_at) = next_probe(&scheduler, &store, at); + assert_eq!(probe, sessions[0]); + waits.push(started_at - at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; } + assert_eq!(waits, [0, 15_000, 30_000, 60_000]); assert_eq!(recovery(&store, &sessions[0]).attempt, 0); - - // An address appears, and the probe starts on that tick. - network.set_addresses(WIFI); - assert_eq!(tick_with_triggers(&scheduler, &store, at), sessions); - - // Ten minutes without a network doubled nothing: this is the first - // failed probe, and the next one is the base delay away. - fail_on_network(&store, &scheduler, &sessions[0], at); - assert!(tick_with_triggers(&scheduler, &store, at + BASE_PROBE_DELAY_MS - 1).is_empty()); - assert_eq!( - tick_with_triggers(&scheduler, &store, at + BASE_PROBE_DELAY_MS), - sessions - ); } +/// Losing every address is a change to an empty fingerprint, which resets +/// nothing, and a probe that comes due meanwhile still starts. #[test] -fn losing_every_address_blocks_a_probe_that_was_due() { +fn losing_every_address_neither_resets_the_delay_nor_blocks_a_due_probe() { let store = Arc::new(Store::in_memory().unwrap()); let network = FakeNetwork::with_addresses(WIFI); let scheduler = network.scheduler(); let now = crate::store::now_timestamp(); - let sessions = network_paused_sessions(&store, &scheduler, 1, now); - assert_eq!(tick_with_triggers(&scheduler, &store, now), sessions); - fail_on_network(&store, &scheduler, &sessions[0], now); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + // The next probe is two minutes off. The addresses go: no early probe. network.set_addresses(&[]); - assert!(tick_with_triggers(&scheduler, &store, now + TICK_MS).is_empty()); - assert!(tick_with_triggers(&scheduler, &store, now + 2 * TICK_MS).is_empty()); + let due = last + 120_000; + let mut at = last + TICK_MS; + while at < due { + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + at += TICK_MS; + } + // Due, with still no address: it starts all the same. + assert_eq!(tick_with_triggers(&scheduler, &store, due).len(), 1); +} + +/// An address appearing after there was none is a change that does reset. +#[test] +fn an_address_appearing_resets_the_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(&[]); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + assert!(tick_with_triggers(&scheduler, &store, last + TICK_MS).is_empty()); network.set_addresses(WIFI); assert_eq!( - tick_with_triggers(&scheduler, &store, now + 3 * TICK_MS), - sessions + tick_with_triggers(&scheduler, &store, last + 2 * TICK_MS).len(), + 1 ); } +/// A wake with no network has nothing new to probe: the schedule stands. #[test] -fn an_unreadable_fingerprint_does_not_block_the_probe() { +fn a_wake_with_no_network_leaves_the_probe_schedule_alone() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + let due = last + 120_000; + + let woke = last + TICK_MS + SUSPENDED_MS; + let mut at = woke; + while at < due { + assert!( + tick_with_triggers(&scheduler, &store, at).is_empty(), + "{} ms before the probe is due", + due - at + ); + at += TICK_MS; + } + assert_eq!(tick_with_triggers(&scheduler, &store, at).len(), 1); +} + +/// A read that fails says nothing about the network, so it does not hold a +/// wake back the way an empty fingerprint does. +#[test] +fn an_unreadable_fingerprint_does_not_hold_back_a_wake() { let store = Arc::new(Store::in_memory().unwrap()); let network = FakeNetwork::with_addresses(&[]); let scheduler = network.scheduler(); let now = crate::store::now_timestamp(); - let sessions = network_paused_sessions(&store, &scheduler, 1, now); - assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); network.make_unreadable(); + let woke = last + TICK_MS + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); assert_eq!( - tick_with_triggers(&scheduler, &store, now + TICK_MS), - sessions + tick_with_triggers(&scheduler, &store, woke + TICK_MS).len(), + 1 ); } +/// The whole path for a lone session behind a dead proxy: the machine is +/// awake with an address, every probe fails, and after the twentieth the +/// session is left paused for a manual resume. +#[test] +fn a_lone_session_gives_up_after_twenty_failed_probes_with_a_network() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + for failure in 1..=MAX_CONSECUTIVE_PROBE_FAILURES { + let (probe, started_at) = next_probe(&scheduler, &store, at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + if failure < MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!(state.gave_up_at, None, "failure {failure}"); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + } else { + assert_eq!(state.gave_up_at, Some(at)); + assert_eq!(state.next_retry_at, None); + assert_eq!(state.reason.as_deref(), Some("probe_limit_exceeded")); + } + } + + // Still paused, and never probed with again. + let row = store.get_session(&sessions[0]).unwrap().unwrap(); + assert_eq!(row.status, SessionStatus::Paused); + let end = at + 2 * MAX_PROBE_DELAY_MS; + while at < end { + at += TICK_MS; + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + } +} + +/// With no network the same run of failures gives up on nothing. +#[test] +fn failed_probes_with_no_network_never_give_up() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + for _ in 0..MAX_CONSECUTIVE_PROBE_FAILURES + 5 { + let (probe, started_at) = next_probe(&scheduler, &store, at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; + } + let state = recovery(&store, &sessions[0]); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("network_wait")); +} + #[test] fn the_fingerprint_is_read_only_while_closed_with_network_rows() { let store = Arc::new(Store::in_memory().unwrap()); diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index da9faf8fd..1affb0c26 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -1914,6 +1914,13 @@ pub(crate) async fn resume_session_for_store( if !transitioned { return Err(SESSION_ALREADY_RUNNING_ERROR.to_string()); } + // A manual resume of a network-paused session doubles as the + // network gate's probe, so getting through resumes the rest. + if let Some(scheduler) = + app_handle.try_state::>() + { + scheduler.manual_resume_started(&session, store::now_timestamp()); + } } // The caller's claim predates `prepare_resume`, and the hold that // keeps the row live is invisible to a Stop: one that landed during diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index 51959ca41..4bdf42da0 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -1794,6 +1794,7 @@ pub fn start_session( // connection, so a remote turn may still be running behind it. Reset // with `settle_reason`. let cancel_unacknowledged = Cell::new(false); + let prompt_failed = Cell::new(false); // Whether the last connection settled out of its background hold with // the work still going (see `SessionSettled::background_work_left`). // Reset with `settle_reason`. @@ -2385,6 +2386,7 @@ pub fn start_session( settle_reason.set(None); transport_loss.set(None); cancel_unacknowledged.set(false); + prompt_failed.set(false); background_work_left.set(false); reattached.set(false); let turn_result = match driver @@ -2452,6 +2454,7 @@ pub fn start_session( settle_reason.set(Some(settled.reason)); transport_loss.set(settled.transport_loss); cancel_unacknowledged.set(settled.cancel_unacknowledged); + prompt_failed.set(settled.prompt_failed); background_work_left.set(settled.background_work_left); match settled.reason { acp_client::SessionSettleReason::HeldUntilCap => { @@ -2547,6 +2550,7 @@ pub fn start_session( settle_reason.set(settlement.settle_reason); transport_loss.set(settlement.transport_loss); cancel_unacknowledged.set(settlement.cancel_unacknowledged); + prompt_failed.set(settlement.prompt_failed); background_work_left.set(settlement.background_work_left); reattached.set(true); // The agent went on with the turn after anything read @@ -2758,6 +2762,7 @@ pub fn start_session( transport_loss: transport_loss.get(), cancel_unacknowledged: cancel_unacknowledged.get(), background_work_left: background_work_left.get(), + prompt_failed: prompt_failed.get(), }; let reattached = reattached.get(); @@ -3020,6 +3025,10 @@ struct RunSettlement { /// The connection settled out of its background hold with the work still /// going (see `SessionSettled::background_work_left`). background_work_left: bool, + /// The agent answered the run's last prompt with an error of its own + /// (see `SessionSettled::prompt_failed`). A turn reattached to was not + /// prompted by the connection that saw it end, so it has the default. + prompt_failed: bool, } /// The transport verdict of a turn reattached to after its connection went: @@ -3319,6 +3328,7 @@ fn finish_session_run( session_snapshot.as_ref(), config.provider.as_deref(), message, + settlement.prompt_failed, crate::store::now_timestamp(), ); match (decision.network_failure, decision.recovery.as_ref()) { @@ -11807,6 +11817,7 @@ draft file" transport_loss: None, cancel_unacknowledged, background_work_left: false, + prompt_failed: false, } } diff --git a/apps/staged/src-tauri/src/session_runner/failed_run.rs b/apps/staged/src-tauri/src/session_runner/failed_run.rs index ec15dffcf..c3efb3b93 100644 --- a/apps/staged/src-tauri/src/session_runner/failed_run.rs +++ b/apps/staged/src-tauri/src/session_runner/failed_run.rs @@ -45,11 +45,13 @@ impl FailedRunDecision { /// /// `session` is the row as it stands after the run, when it could be read; /// `fallback_provider` is the run's configured provider, for a row that -/// records none. +/// records none; `prompt_failed` is the driver's flag that the agent answered +/// the prompt with an error (`acp_client::SessionSettled::prompt_failed`). pub(super) fn decide_failed_run( session: Option<&Session>, fallback_provider: Option<&str>, driver_error: &str, + prompt_failed: bool, now_millis: i64, ) -> FailedRunDecision { let provider_metadata = session.and_then(|session| session.provider_metadata.as_ref()); @@ -74,7 +76,7 @@ pub(super) fn decide_failed_run( let network_failure = if is_pipeline { None } else { - classify_network_failure(provider, driver_error, provider_metadata) + classify_network_failure(provider, prompt_failed, provider_metadata) }; if network_failure.is_some() { return FailedRunDecision { @@ -138,7 +140,13 @@ mod tests { fn a_network_prompt_error_pauses_with_the_network_kind_and_no_reset() { let session = session_with_dns_failure(); - let decision = decide_failed_run(Some(&session), None, &driver_error(DNS_MESSAGE), NOW_MS); + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); assert_eq!(decision.status(), "paused"); assert_eq!(decision.completion_reason, CompletionReason::RateLimited); @@ -164,7 +172,8 @@ mod tests { store.create_session(&session).unwrap(); let row = store.get_session(&session.id).unwrap().unwrap(); - let decision = decide_failed_run(Some(&row), None, &driver_error(DNS_MESSAGE), NOW_MS); + let decision = + decide_failed_run(Some(&row), None, &driver_error(DNS_MESSAGE), true, NOW_MS); let recovery = decision.recovery.expect("a network failure pauses"); assert!(crate::rate_limit_scheduler::pause_for_rate_limit( &store, @@ -205,8 +214,13 @@ mod tests { let mut session = session_with_dns_failure(); session.pipeline = Some(PipelineExecution::from_steps(&[]).with_kind(kind)); - let decision = - decide_failed_run(Some(&session), None, &driver_error(DNS_MESSAGE), NOW_MS); + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); assert_eq!(decision.status(), "error"); assert_eq!(decision.completion_reason, CompletionReason::Crashed); @@ -221,12 +235,12 @@ mod tests { session.provider = None; let error = driver_error(DNS_MESSAGE); - let with_fallback = decide_failed_run(Some(&session), Some("claude"), &error, NOW_MS); + let with_fallback = decide_failed_run(Some(&session), Some("claude"), &error, true, NOW_MS); assert_eq!(with_fallback.status(), "paused"); // Codex is never classified, and neither is an unknown agent. for provider in [Some("codex"), None] { - let decision = decide_failed_run(Some(&session), provider, &error, NOW_MS); + let decision = decide_failed_run(Some(&session), provider, &error, true, NOW_MS); assert_eq!(decision.status(), "error", "{provider:?}"); assert_eq!(decision.completion_reason, CompletionReason::Crashed); } @@ -245,6 +259,7 @@ mod tests { Some(&session), None, &driver_error("Internal error: rate limit exceeded"), + true, NOW_MS, ); @@ -257,18 +272,44 @@ mod tests { assert_eq!(decision.network_failure, None); } + /// The same row and error text, but the driver did not report a failed + /// prompt: the stored `promptError` is stale and the run fails. + #[test] + fn a_run_that_did_not_fail_on_the_prompt_is_not_paused() { + let session = session_with_dns_failure(); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + false, + NOW_MS, + ); + + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + assert_eq!(decision.network_failure, None); + } + #[test] fn other_failures_still_fail_the_session() { // A real 5xx: the same structured kind, no network token. let mut session = session_with_dns_failure(); let message = "Internal error: API Error: 500 Internal server error"; session.provider_metadata = Some(prompt_error_metadata("server_error", message)); - let decision = decide_failed_run(Some(&session), None, &driver_error(message), NOW_MS); + let decision = + decide_failed_run(Some(&session), None, &driver_error(message), true, NOW_MS); assert_eq!(decision.status(), "error"); assert_eq!(decision.completion_reason, CompletionReason::Crashed); // A row that could not be read has nothing to classify. - let decision = decide_failed_run(None, Some("claude"), &driver_error(DNS_MESSAGE), NOW_MS); + let decision = decide_failed_run( + None, + Some("claude"), + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); assert_eq!(decision.status(), "error"); assert_eq!(decision.completion_reason, CompletionReason::Crashed); @@ -277,6 +318,7 @@ mod tests { None, Some("claude"), "ACP protocol failed: server shut down unexpectedly", + false, NOW_MS, ); assert_eq!(decision.status(), "error"); diff --git a/apps/staged/src-tauri/src/session_runner/network_failure.rs b/apps/staged/src-tauri/src/session_runner/network_failure.rs index c92b3f48a..0408a6b8a 100644 --- a/apps/staged/src-tauri/src/session_runner/network_failure.rs +++ b/apps/staged/src-tauri/src/session_runner/network_failure.rs @@ -15,8 +15,6 @@ //! [`matched_fallback_signal`] below. **No other file may match error prose //! to decide whether a failure was the network.** -use acp_client::PROMPT_FAILED_PREFIX; - /// The JSON-RPC "internal error" code. Both bridges report a failed provider /// request with it; `-32000` (authentication required) and every other code /// are not network failures. @@ -64,13 +62,16 @@ impl NetworkFailure { /// Classify a failed run as a network failure, or `None` for anything else. /// -/// `driver_error` is the run's error as the driver reported it and -/// `provider_metadata` the session's stored metadata. The checks run in this -/// order, and only the last one reads error prose: +/// `prompt_failed` is the driver's own flag for the run +/// (`acp_client::SessionSettled::prompt_failed`) and `provider_metadata` the +/// session's stored metadata. The checks run in this order, and only the +/// last one reads error prose: /// -/// 1. The run failed on the prompt. `promptError` can be left on the row by an -/// earlier event of the same turn, so a failure that is not a prompt -/// failure must not be judged by it. +/// 1. The run failed on the prompt: the agent answered `session/prompt` with +/// an error over a live transport. `promptError` can be left on the row by +/// an earlier event of the same turn, so a failure during setup, after a +/// cancel or an interrupt, or on a lost transport must not be judged by +/// it. The driver reports this as a flag, so no error text is parsed. /// 2. The error code is `-32603`. /// 3. `data.reason` is not `incoming_transport_closed`. /// 4. The agent's own structured field allows a network cause. @@ -80,11 +81,11 @@ impl NetworkFailure { /// still run the rate-limit check first. pub(crate) fn classify_network_failure( provider: Option<&str>, - driver_error: &str, + prompt_failed: bool, provider_metadata: Option<&serde_json::Value>, ) -> Option { let agent = NetworkAgent::from_provider(provider?)?; - if !failed_on_the_prompt(driver_error) { + if !prompt_failed { return None; } let prompt_error = provider_metadata?.pointer("/acp/promptError")?; @@ -131,25 +132,6 @@ impl NetworkAgent { } } -/// Whether the driver reported a failed `session/prompt`. -/// -/// The driver's text starts with its own [`PROMPT_FAILED_PREFIX`]. The -/// connection that carried the prompt usually hands it on wrapped once, as -/// `ACP protocol failed: Error { ..., data: Some(String("Prompt failed: ...")) }`, -/// so the outermost `data` string is accepted too. Both forms are Staged's -/// own rendering, not agent prose. A deeper occurrence does not count: that -/// would be an agent quoting the words. -fn failed_on_the_prompt(driver_error: &str) -> bool { - if driver_error.starts_with(PROMPT_FAILED_PREFIX) { - return true; - } - // Outermost first, and only an unescaped (top-level) string counts. - const WRAPPED: &str = "data: Some(String(\""; - driver_error - .find(WRAPPED) - .is_some_and(|at| driver_error[at + WRAPPED.len()..].starts_with(PROMPT_FAILED_PREFIX)) -} - // --------------------------------------------------------------------------- // The fallback table: the only place that reads error prose. // @@ -260,27 +242,8 @@ mod tests { metadata(json!({ "code": -32603, "message": message, "data": { "reason": "turn_failed" } })) } - /// The driver error as the runner receives it today: the prompt failure - /// wrapped once by the connection that carried it. - fn wrapped_driver_error(message: &str) -> String { - let inner = format!( - r#"Prompt failed: Error {{ code: -32603: Internal error, message: {message:?}, data: None }}"# - ); - format!( - r#"ACP protocol failed: Error {{ code: -32603: Internal error, message: "Internal error", data: Some(String({inner:?})) }}"# - ) - } - fn classify(provider: &str, metadata: &serde_json::Value) -> Option { - let message = metadata - .pointer("/acp/promptError/message") - .and_then(|m| m.as_str()) - .unwrap_or_default(); - classify_network_failure( - Some(provider), - &wrapped_driver_error(message), - Some(metadata), - ) + classify_network_failure(Some(provider), true, Some(metadata)) } #[test] @@ -319,21 +282,6 @@ mod tests { } } - #[test] - fn the_bare_and_the_wrapped_driver_error_both_count_as_a_prompt_failure() { - let metadata = claude_error("server_error", DNS_MESSAGE); - let bare = format!( - r#"Prompt failed: Error {{ code: -32603: Internal error, message: {DNS_MESSAGE:?}, data: Some(Object {{"errorKind": String("server_error")}}) }}"# - ); - for driver_error in [bare, wrapped_driver_error(DNS_MESSAGE)] { - assert_eq!( - classify_network_failure(Some("claude"), &driver_error, Some(&metadata)), - Some(NetworkFailure::ClaudeErrno("ENOTFOUND")), - "{driver_error}" - ); - } - } - #[test] fn a_real_5xx_without_a_token_is_not_a_network_failure() { for message in [ @@ -469,37 +417,29 @@ mod tests { } } + /// A `promptError` left on the row by an earlier event says nothing + /// about a run the driver did not report as a failed prompt, whatever + /// the stored error looks like. #[test] fn a_stale_prompt_error_is_ignored_when_the_run_did_not_fail_on_the_prompt() { - let metadata = claude_error("server_error", DNS_MESSAGE); - for driver_error in [ - "Timed out waiting for ACP protocol startup after 90s".to_string(), - "ACP protocol failed: server shut down unexpectedly".to_string(), - format!(r#"Prompt failed after interrupt: Error {{ message: {DNS_MESSAGE:?} }}"#), - format!(r#"Prompt failed after cancellation: Error {{ message: {DNS_MESSAGE:?} }}"#), - r#"ACP protocol failed: Error { code: -32603: Internal error, message: "Internal error", data: Some(String("ACP authentication is required to run the prompt. Sign this agent in, then retry.")) }"#.to_string(), - // The words quoted deeper down are an agent's, not the driver's. - r#"ACP protocol failed: Error { code: -32603: Internal error, message: "Internal error", data: Some(String("Failed to load session: Error { data: Some(String(\"Prompt failed: nope\")) }")) }"#.to_string(), - r#"Failed to load session: Error { message: "Prompt failed: (ENOTFOUND)", data: None }"#.to_string(), - String::new(), + for (provider, metadata) in [ + ("claude", claude_error("server_error", DNS_MESSAGE)), + ("goose", goose_error("Network error: connection reset")), ] { + assert!(classify(provider, &metadata).is_some()); assert_eq!( - classify_network_failure(Some("claude"), &driver_error, Some(&metadata)), + classify_network_failure(Some(provider), false, Some(&metadata)), None, - "{driver_error}" + "{provider}" ); } } #[test] fn a_prompt_failure_without_a_stored_prompt_error_is_not_classified() { - let driver_error = wrapped_driver_error(DNS_MESSAGE); - assert_eq!( - classify_network_failure(Some("claude"), &driver_error, None), - None - ); + assert_eq!(classify_network_failure(Some("claude"), true, None), None); assert_eq!( - classify_network_failure(Some("claude"), &driver_error, Some(&json!({ "acp": {} }))), + classify_network_failure(Some("claude"), true, Some(&json!({ "acp": {} }))), None ); } @@ -557,10 +497,9 @@ mod tests { fn codex_and_unknown_providers_are_never_classified() { let mut metadata = claude_error("server_error", DNS_MESSAGE); metadata["acp"]["promptError"]["data"]["reason"] = json!("turn_failed"); - let driver_error = wrapped_driver_error(DNS_MESSAGE); for provider in [Some("codex"), Some("amp"), Some(""), None] { assert_eq!( - classify_network_failure(provider, &driver_error, Some(&metadata)), + classify_network_failure(provider, true, Some(&metadata)), None, "{provider:?}" ); diff --git a/apps/staged/src-tauri/src/store/messages.rs b/apps/staged/src-tauri/src/store/messages.rs index 1dbe4f08c..2645a71f3 100644 --- a/apps/staged/src-tauri/src/store/messages.rs +++ b/apps/staged/src-tauri/src/store/messages.rs @@ -700,25 +700,35 @@ impl Store { .map_err(Into::into) } - /// Whether a session recorded agent output at or after `since`: text, a - /// tool call or its result, a plan, or a turn's response. The rows counted - /// are those of [`Self::agent_answered_message`], bounded by time instead - /// of by a message id. + /// Whether a session's agent reached its model provider at or after + /// `since`: the session recorded a tool call or its result, a result the + /// agent did not mark as an error, or a prompt response. /// /// Backs the network gate's probe check, which knows when it claimed the - /// probe but not which row its prompt will be. - pub fn session_has_agent_output_since( + /// probe but not which row its prompt will be. It is narrower than + /// [`Self::agent_answered_message`] on purpose. A turn that fails because + /// the provider is unreachable still records agent output: Claude prints + /// the failure as a message chunk ("API Error: Can't reach the API + /// server"), then a `response_result` with `is_error: true`, and the + /// writer stores that text as a plain assistant row. None of those shows + /// that a request got through, so message text and plans never count + /// here and a result counts only without the error mark. + pub fn session_reached_provider_since( &self, session_id: &str, since: i64, ) -> Result { let conn = self.conn.lock().unwrap(); conn.query_row( - &format!( - "SELECT EXISTS( - SELECT 1 FROM session_messages - WHERE session_id = ?1 AND created_at >= ?2 AND {AGENT_OUTPUT_FILTER})" - ), + "SELECT EXISTS( + SELECT 1 FROM session_messages + WHERE session_id = ?1 AND created_at >= ?2 AND acp_origin IS NULL + AND (role IN ('tool_call', 'tool_result') + OR (role = 'assistant' AND acp_event_kind = 'prompt_response') + OR (role = 'assistant' AND acp_event_kind = 'response_result' + AND CASE WHEN json_valid(acp_content) + THEN json_extract(acp_content, '$.is_error') = 0 + ELSE 0 END)))", params![session_id, since], |row| row.get(0), ) diff --git a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts index 2e6add9c8..3ff9e7acd 100644 --- a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts +++ b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts @@ -115,9 +115,7 @@ describe('projectNoteRow', () => { hour: 'numeric', minute: '2-digit', }); - expect(paused({ rateLimitKind: 'network' }).title).toBe( - 'Paused: network unavailable. Resumes when the connection returns.' - ); + expect(paused({ rateLimitKind: 'network' }).title).toBe('Paused: network unavailable.'); expect(paused({}).title).toBe('Paused by API limit — no note yet'); // The reset time dates the pause; it is not the session's error message. expect(paused({ rateLimitResetsAt: resetsAt }).title).toBe( diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts index 12a33beeb..b5abed360 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts @@ -434,7 +434,29 @@ describe('newestUserMessageId', () => { ); }); - it('says when a network pause tries again', () => { + it('gives no retry time for a network pause waiting behind the gate', () => { + // As the scheduler writes it: `nextRetryAt` is when the row paused, so + // it is already in the past and stays there for the whole outage. + const pausedAt = Date.UTC(2026, 5, 24, 15, 45); + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { + staged: { + rateLimitAutoRecovery: { + attempt: 0, + maxAttempts: 8, + nextRetryAt: pausedAt, + reason: 'network_wait', + }, + }, + }, + }) + ).toBe('Paused: network unavailable. Resumes when the connection returns.'); + }); + + it('says when a network pause on its own backoff tries again', () => { const nextRetryAt = Date.UTC(2026, 5, 24, 15, 45); const retryTime = new Date(nextRetryAt).toLocaleTimeString([], { hour: 'numeric', @@ -445,7 +467,9 @@ describe('newestUserMessageId', () => { completionReason: 'rate_limited', rateLimitKind: 'network', providerMetadata: { - staged: { rateLimitAutoRecovery: { attempt: 2, maxAttempts: 8, nextRetryAt } }, + staged: { + rateLimitAutoRecovery: { attempt: 2, maxAttempts: 8, nextRetryAt, reason: 'backoff' }, + }, }, }) ).toBe( @@ -456,6 +480,7 @@ describe('newestUserMessageId', () => { it('asks for a manual resume of a network pause that will not resume by itself', () => { for (const rateLimitAutoRecovery of [ { attempt: 9, maxAttempts: 8, gaveUpAt: Date.UTC(2026, 5, 24, 15, 45) }, + { attempt: 0, gaveUpAt: Date.UTC(2026, 5, 24, 15, 45), reason: 'probe_limit_exceeded' }, { unsafeToAutoResume: true, reason: 'pending_tool_call' }, ]) { expect( diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts index 8e12cfb6b..cb9b8c084 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts @@ -10,7 +10,8 @@ import { } from '../../shared/disconnectedSession'; import { NETWORK_PAUSED, - NETWORK_PAUSED_SUBTITLE, + NETWORK_PAUSED_RESUMES, + NETWORK_WAIT_REASON, formatRateLimitResetTime, isNetworkPause, } from '../timeline/sessionFailureCopy'; @@ -289,15 +290,19 @@ function rateLimitEndMessage( * The notice for a network pause. A row the scheduler gave up on, or will not * resume by itself, no longer resumes when the connection returns, so it says * what to do instead of repeating that promise. + * + * A row waiting behind the scheduler's network gate gets no retry time: its + * `nextRetryAt` is the time it paused, and the gate's next probe time is not + * exposed. Only a pause on the session's own backoff has a time to show. */ function networkPauseEndMessage(recovery: RateLimitAutoRecovery | null): string { if (recovery?.gaveUpAt || recovery?.unsafeToAutoResume) { return `${NETWORK_PAUSED} Resume manually when ready.`; } - if (recovery?.nextRetryAt) { - return `${NETWORK_PAUSED_SUBTITLE} Trying again around ${formatTimestamp(recovery.nextRetryAt)}.`; + if (recovery?.nextRetryAt && recovery.reason !== NETWORK_WAIT_REASON) { + return `${NETWORK_PAUSED_RESUMES} Trying again around ${formatTimestamp(recovery.nextRetryAt)}.`; } - return NETWORK_PAUSED_SUBTITLE; + return NETWORK_PAUSED_RESUMES; } /** @@ -345,6 +350,7 @@ interface RateLimitAutoRecovery { nextRetryAt?: number | null; gaveUpAt?: number | null; unsafeToAutoResume?: boolean; + reason?: string | null; } function rateLimitAutoRecovery( diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts index 1a8be5817..4ef36ae09 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts @@ -107,11 +107,13 @@ describe('failedArtifactSubtitle', () => { ); }); - it('says a network pause waits for the connection, whatever the artifact', () => { + // The row cannot know whether the scheduler gave up on the pause or will + // not resume it by itself, so it must not say the session resumes. + it('names a network pause without promising a resume, whatever the artifact', () => { for (const kind of ['commit', 'note', 'review'] as const) { - expect(failedArtifactSubtitle('rate_limited', kind, { rateLimitKind: 'network' })).toBe( - 'Paused: network unavailable. Resumes when the connection returns.' - ); + const subtitle = failedArtifactSubtitle('rate_limited', kind, { rateLimitKind: 'network' }); + expect(subtitle).toBe('Paused: network unavailable.'); + expect(subtitle).not.toMatch(/resume/i); } }); @@ -121,7 +123,7 @@ describe('failedArtifactSubtitle', () => { rateLimitKind: 'network', rateLimitResetsAt: '2026-06-24T15:45:00Z', }) - ).toBe('Paused: network unavailable. Resumes when the connection returns.'); + ).toBe('Paused: network unavailable.'); }); it('reads the network kind only on a rate-limit pause', () => { diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts index 19a94fc65..429053632 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts @@ -31,9 +31,21 @@ const RECONCILE_INTERRUPTIONS: ReadonlySet = new Set([ */ export const NETWORK_PAUSE_KIND = 'network'; +/** + * The `reason` in a network pause's retry bookkeeping while it waits behind + * the scheduler's network gate (`NETWORK_WAIT_REASON` in + * `src-tauri/src/rate_limit_scheduler.rs`). Such a row's `nextRetryAt` is the + * time it paused, not the time of its next attempt. + */ +export const NETWORK_WAIT_REASON = 'network_wait'; + /** What a network pause says wherever it is named: the row subtitle and the chat alert. */ export const NETWORK_PAUSED = 'Paused: network unavailable.'; -export const NETWORK_PAUSED_SUBTITLE = `${NETWORK_PAUSED} Resumes when the connection returns.`; +/** + * The same, with the promise of an automatic resume. Only for a caller that + * has the scheduler's bookkeeping and so knows the promise holds. + */ +export const NETWORK_PAUSED_RESUMES = `${NETWORK_PAUSED} Resumes when the connection returns.`; /** Whether a `rate_limited` pause with this `rateLimitKind` is a network pause. */ export function isNetworkPause(rateLimitKind: string | null | undefined): boolean { @@ -79,7 +91,11 @@ export function failedArtifactSubtitle( case 'remote_agent_lost': return `Workspace agent lost — no ${noun} created`; case 'rate_limited': - if (isNetworkPause(rateLimitKind)) return NETWORK_PAUSED_SUBTITLE; + // A timeline row does not carry the scheduler's bookkeeping, so it + // cannot tell a pause that will resume by itself from one the + // scheduler gave up on or will not resume (a pending tool call). It + // makes no promise either way; the chat alert, which can tell, does. + if (isNetworkPause(rateLimitKind)) return NETWORK_PAUSED; return rateLimitPausedSubtitle(rateLimitResetsAt) ?? `Paused by API limit — no ${noun} yet`; case 'app_quit': return `Session interrupted — no ${noun} created`; diff --git a/crates/acp-client/src/driver.rs b/crates/acp-client/src/driver.rs index cee4447b0..2cf70da3b 100644 --- a/crates/acp-client/src/driver.rs +++ b/crates/acp-client/src/driver.rs @@ -1052,6 +1052,14 @@ pub struct SessionSettled { /// after a Stop cancel it again over a fresh connection /// ([`AcpDriver::attach_cancel`]). pub background_work_left: bool, + /// The agent answered the last turn's `session/prompt` with an error of + /// its own, over a transport that was still up to carry it. `false` for + /// a failure during setup, after a cancel or an interrupt, and for one + /// the SDK made because the transport closed. + /// + /// This is the structured form of "the run failed on the prompt": a + /// caller that needs to know it reads this instead of the error text. + pub prompt_failed: bool, } impl SessionSettled { @@ -3155,6 +3163,7 @@ impl AcpDriver { transport_loss, cancel_unacknowledged: transport.cancel_unacknowledged(), background_work_left: transport.background_work_left(), + prompt_failed: transport.answered_error(), }); if let Some(reply) = pending_reply.borrow_mut().take() { @@ -7776,13 +7785,9 @@ async fn record_prompt_error( /// How the driver's error text begins when the agent answered `session/prompt` /// with an error other than `auth_required` (see [`describe_prompt_error`]). -/// -/// This is the driver's own wording, not the agent's, so a caller may test for -/// it to learn that a run failed on the prompt itself and not during setup, -/// after a cancel, or on a lost transport. The colon is part of the prefix: -/// "Prompt failed after interrupt: ..." and "Prompt failed after -/// cancellation: ..." are different failures and do not match. -pub const PROMPT_FAILED_PREFIX: &str = "Prompt failed:"; +/// Display text only: a caller that needs to know a run failed on the prompt +/// reads [`SessionSettled::prompt_failed`]. +const PROMPT_FAILED_PREFIX: &str = "Prompt failed:"; /// The text a failed `session/prompt` surfaces to the user. /// @@ -9329,6 +9334,61 @@ mod tests { ); } + /// [`SessionSettled::prompt_failed`] is read off the transport monitor, + /// which the turn marks when the agent itself answers the prompt with an + /// error. The session runner classifies a failed run from that flag, so + /// it must be set by a real answered error and by nothing before it. + #[tokio::test(flavor = "current_thread")] + async fn an_answered_prompt_error_marks_the_prompt_failed() { + let provider_error = agent_client_protocol::Error::internal_error() + .data(serde_json::json!({ "errorKind": "server_error" })); + let agent = agent_client_protocol::Agent.builder().on_receive_request( + async move |_request: PromptRequest, responder, _cx| { + responder.respond_with_error(provider_error.clone()) + }, + agent_client_protocol::on_receive_request!(), + ); + let setup = AcpSessionSetup { + agent_session_id: "session-1".to_string(), + agent_capabilities: AgentCapabilities::default(), + task_tracking_mode: TaskTrackingMode::Raw, + auth_methods: vec![], + steering_supported: false, + }; + let handler = Arc::new( + AcpNotificationHandler::new( + Arc::new(BasicMessageWriter::new()), + false, + vec![], + CancellationToken::new(), + ) + .named("sess-1"), + ); + assert!(!handler.transport.answered_error()); + + let (_turn_control_tx, mut turn_control_rx) = mpsc::unbounded_channel(); + let outcome = agent_client_protocol::Client + .connect_with(agent, async |connection| { + Ok(run_prompt_turn( + &connection, + &setup, + &handler, + "local-session", + &CancellationToken::new(), + &mut turn_control_rx, + "hello", + &[], + ) + .await) + }) + .await + .expect("the connection itself stays healthy"); + + let error = outcome.expect_err("the agent's error should fail the prompt"); + assert!(error.starts_with("Prompt failed: "), "{error}"); + assert!(handler.transport.answered_error()); + } + /// A `-32000` with the bridge's own sentence on it, as pi-acp sends at /// `session/new`; the stock `auth_required` is what every other surveyed /// bridge sends. @@ -14057,6 +14117,7 @@ agent: http=false, sse=false). Select a provider that supports MCP over HTTP/SSE transport_loss: None, cancel_unacknowledged: false, background_work_left: false, + prompt_failed: false, }; // A hold that was cancelled or failed after the turn completed is the diff --git a/crates/acp-client/src/lib.rs b/crates/acp-client/src/lib.rs index cbceaac66..ca4a69624 100644 --- a/crates/acp-client/src/lib.rs +++ b/crates/acp-client/src/lib.rs @@ -43,7 +43,6 @@ pub use driver::{ RemoteProcessStatus, ReplayBoundary, SessionConnection, SessionLifetime, SessionSettleReason, SessionSettled, SteerError, SteerOutcome, Store, TransportLoss, TransportLossPhase, TurnControlHandle, TurnResult, BACKGROUND_CONTINUATION_ORIGIN, INTERRUPT_ACK_TIMEOUT, - PROMPT_FAILED_PREFIX, }; pub use simple::{run_acp_prompt, run_acp_prompt_with_interpreter_env_snapshot}; pub use types::{ From a36de9477729654df5191fa155481a285d61b9dc Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 16:59:52 +1100 Subject: [PATCH 8/8] fix(staged): bound late probe failures and restart the network window Resolves the two warnings of the review of dddf0597. Timed-out probe. A probe that failed on the network only after `PROBE_TIMEOUT_MS` escaped the bound of 20 failed probes: by then the gate was closed, so the failure was answered `Waiting` without touching the session's count, and its row, due at once, was probed again on the next tick. A lone session could repeat that without limit. The gate now remembers each timed-out probe in `timed_out_probes`, with whether the process was suspended during the turn, before or after the timeout. When that session later fails on the network: - the failure is charged to its run of failed probes if the machine stayed awake and the last fingerprint held an address, the conditions of a probe that fails in time, and the twentieth gives up with `probe_limit_exceeded`; - a closed gate waits at least the current probe delay from the failure before its next probe. The delay is not doubled again, because the timeout already doubled it for this probe; - a probe another session is running is left alone. The record is dropped when the session fails or probes again, and all records are cleared when the gate opens, with the failure counts. A late failure with the gate open is counted as before. The comment on `PROBE_TIMEOUT_MS` claimed it was longer than a provider request takes to time out. That was never measured. It now says so, and that a probe may fail after the timeout and is then charged as above. 24 hour window. For a network pause the window is now measured only across counted failures. `first_limited_at` was set again only when `attempt` became 1, so the session that closed the gate (counted once), then waited behind it for more than a day, gave up with `retry_window_elapsed` on its second counted failure. A counted failure now restarts the window when the pause before it was not counted, read from the bookkeeping `reason` that pause left (`network_wait` or `probe_limit_exceeded`). No field is added. The attempt cap is not restarted, and counted failures with no wait between them still give up once they span more than 24 hours. API-limit pauses are unchanged. Tests, each failing on dddf0597 except the two noted: twenty late probe failures exhaust a session; a late failure holds the next probe for the probe delay without doubling it, at the gate and through the scheduler for a lone session; a late failure is charged only awake with a network present, and is charged while another session probes without disturbing it; a wait between two counted failures restarts the window. Passing before and after: an opened gate forgets timed-out probes, a late failure never brings a later probe time forward, and waits do not restart the attempt cap. `the_network_window_runs_from_the_first_counted_failure` asserted the old rule and is replaced by `the_network_window_runs_across_counted_failures`. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 --- .../src-tauri/src/rate_limit_scheduler.rs | 92 +++++++-- .../src/rate_limit_scheduler/network_gate.rs | 67 ++++++- .../network_gate/tests.rs | 184 ++++++++++++++++++ 3 files changed, 319 insertions(+), 24 deletions(-) diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index 02cca40e3..29c146a6b 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -689,11 +689,14 @@ fn is_actionable_for_auto_resume(session: &Session) -> bool { /// - [`PauseAccounting::ProbesExhausted`]: the row gives up and waits for a /// manual resume. /// - [`PauseAccounting::Counted`]: the rules below, with one difference. The -/// 24 hour window of a network pause runs from its first counted failure, -/// not from its first pause: `first_limited_at` is set again when -/// `attempt` becomes 1. Time spent waiting behind a closed gate (a laptop -/// asleep for a weekend) therefore never ends a session on the first -/// failure that counts. +/// 24 hour window of a network pause is measured only across counted +/// failures, not from its first pause: `first_limited_at` is set again +/// when `attempt` becomes 1, and when the pause before this one was not +/// counted (see [`last_pause_was_uncounted`]). Time spent waiting behind a +/// closed gate (a laptop asleep for a weekend) therefore never ends a +/// session, whether the wait came before its first counted failure or +/// between two of them. Counted failures with no wait between them still +/// give up once they span more than 24 hours. fn build_recovery_state( existing: Option<&RateLimitAutoRecovery>, kind: Option<&str>, @@ -745,7 +748,7 @@ fn build_recovery_state( } let attempt = existing.map(|s| s.attempt.saturating_add(1)).unwrap_or(1); - let first_limited_at = if network && attempt == 1 { + let first_limited_at = if network && (attempt == 1 || last_pause_was_uncounted(existing)) { now } else { first_limited_at @@ -803,6 +806,16 @@ fn build_recovery_state( } } +/// Whether the bookkeeping left by the pause before this one shows that it +/// was a network pause not charged to the retry budget: a wait behind the +/// gate, or the end of the session's probes. The `reason` is that of the +/// latest pause, so no field is needed to tell. +fn last_pause_was_uncounted(existing: Option<&RateLimitAutoRecovery>) -> bool { + existing + .and_then(|state| state.reason.as_deref()) + .is_some_and(|reason| reason == NETWORK_WAIT_REASON || reason == PROBE_LIMIT_REASON) +} + /// The outcome of trying to take ownership of a rate-limited session for one /// auto-resume attempt. enum AutoResumeClaim { @@ -1880,25 +1893,16 @@ mod tests { assert!(counted.next_retry_at.unwrap() > failed_at); } - /// The window still ends a network pause: 24 hours after its first - /// counted failure, the next counted one gives up. Waits in between do - /// not move the start. + /// The window still ends a network pause: counted failures with no wait + /// between them give up once they span more than 24 hours. #[test] - fn the_network_window_runs_from_the_first_counted_failure() { + fn the_network_window_runs_across_counted_failures() { let first_counted_at = 1_000_000; let first = network_state(None, PauseAccounting::Counted, first_counted_at); assert_eq!(first.first_limited_at, first_counted_at); - let waiting = network_state( - Some(&first), - PauseAccounting::Waiting, - first_counted_at + DAY_MS / 2, - ); - assert_eq!(waiting.attempt, 1); - assert_eq!(waiting.first_limited_at, first_counted_at); - let inside = network_state( - Some(&waiting), + Some(&first), PauseAccounting::Counted, first_counted_at + DAY_MS, ); @@ -1907,11 +1911,59 @@ mod tests { assert_eq!(inside.first_limited_at, first_counted_at); let outside_at = first_counted_at + DAY_MS + 1; - let outside = network_state(Some(&waiting), PauseAccounting::Counted, outside_at); + let outside = network_state(Some(&inside), PauseAccounting::Counted, outside_at); + assert_eq!(outside.attempt, 3); assert_eq!(outside.gave_up_at, Some(outside_at)); assert_eq!(outside.reason.as_deref(), Some("retry_window_elapsed")); } + /// The session that closed the gate when the laptop slept: counted once, + /// then waiting behind the gate for more than a day. Its next counted + /// failure, on a flaky connection after the wake, restarts the window + /// and is scheduled, not given up on. + #[test] + fn a_wait_between_two_counted_network_failures_restarts_the_window() { + let first_counted_at = 1_000_000; + let first = network_state(None, PauseAccounting::Counted, first_counted_at); + + let waited_at = first_counted_at + DAY_MS / 2; + let waiting = network_state(Some(&first), PauseAccounting::Waiting, waited_at); + assert_eq!(waiting.attempt, 1); + + let second_counted_at = first_counted_at + 30 * 60 * 60 * 1_000; + let second = network_state(Some(&waiting), PauseAccounting::Counted, second_counted_at); + assert_eq!(second.attempt, 2); + assert_eq!(second.gave_up_at, None); + assert_eq!(second.reason.as_deref(), Some("backoff")); + assert_eq!(second.first_limited_at, second_counted_at); + + // The restarted window runs from that failure. + let inside = network_state( + Some(&second), + PauseAccounting::Counted, + second_counted_at + DAY_MS, + ); + assert_eq!(inside.gave_up_at, None); + assert_eq!(inside.first_limited_at, second_counted_at); + let outside_at = second_counted_at + DAY_MS + 1; + let outside = network_state(Some(&second), PauseAccounting::Counted, outside_at); + assert_eq!(outside.gave_up_at, Some(outside_at)); + assert_eq!(outside.reason.as_deref(), Some("retry_window_elapsed")); + } + + /// The attempt cap is not restarted by a wait. + #[test] + fn a_wait_between_counted_network_failures_keeps_the_attempt_cap() { + let mut state = network_state(None, PauseAccounting::Counted, 1_000_000); + for round in 1..=i64::from(MAX_AUTO_RETRY_ATTEMPTS) { + let now = 1_000_000 + round * 60_000; + state = network_state(Some(&state), PauseAccounting::Waiting, now); + state = network_state(Some(&state), PauseAccounting::Counted, now + 1); + } + assert_eq!(state.attempt, MAX_AUTO_RETRY_ATTEMPTS + 1); + assert_eq!(state.reason.as_deref(), Some("retry_cap_exceeded")); + } + /// An API-limit pause keeps measuring from its first pause. #[test] fn the_api_limit_window_still_runs_from_the_first_pause() { diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs index 28a9fc6e6..2d287c060 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs @@ -34,6 +34,13 @@ //! for a manual resume: with a network evidently present, its failure is not //! one that waiting fixes (a dead local proxy, a captive portal). //! +//! A probe that fails only after the gate stopped waiting for it is still +//! that session's failed probe. The gate remembers each timed-out probe until +//! the gate opens or the session probes again, charges its late network +//! failure to the same bound under the same conditions, and holds the next +//! probe back by the probe delay, so a session that always fails late cannot +//! be probed again and again without limit. +//! //! Two things make a closed gate probe sooner than its doubling delay, both //! derived inside the scheduler tick with no operating-system callbacks: //! @@ -72,8 +79,13 @@ pub(super) const MAX_PROBE_DELAY_MS: i64 = 10 * 60_000; /// three of the scheduler's 15 second ticks. pub(super) const WAKE_GAP_MS: i64 = 45_000; /// How long a probe may run without reaching its provider before the gate -/// stops waiting for it. Longer than a provider request takes to time out, -/// so a probe that will fail has normally failed by then. +/// stops waiting for it, so one hung turn cannot hold the other rows. +/// +/// It is not known to be longer than an agent's own request timeout: that +/// was never measured, and an agent that retries a request that timed out +/// takes longer still. So a probe may fail on the network after this timeout +/// has passed. Such a late failure is charged to the session as a failed +/// probe (see [`NetworkGate::record_failure`]). pub(super) const PROBE_TIMEOUT_MS: i64 = 10 * 60_000; /// How many times in a row one session's own probes may fail, each while the /// machine stayed awake and held a network address, before the session is @@ -123,6 +135,11 @@ pub(super) struct NetworkGate { /// Per session, how many of its own probes failed in a row while the /// machine was awake with a network address. Cleared when the gate opens. probe_failures: HashMap, + /// The sessions whose probe timed out and whose turn has not been heard + /// of since, each with whether the process was suspended during that + /// turn. Cleared when the gate opens; a session leaves when it fails on + /// the network or probes again. + timed_out_probes: HashMap, } impl Default for NetworkGate { @@ -135,6 +152,7 @@ impl Default for NetworkGate { settled: true, addresses: None, probe_failures: HashMap::new(), + timed_out_probes: HashMap::new(), } } } @@ -162,6 +180,9 @@ impl NetworkGate { if let GateState::Probing { slept, .. } = &mut self.state { *slept = true; } + for slept in self.timed_out_probes.values_mut() { + *slept = true; + } } woke } @@ -228,6 +249,9 @@ impl NetworkGate { } fn probe_started(&mut self, session_id: &str, now: i64) { + // A session that probes again was parked since its probe timed out, + // so that turn is over. + self.timed_out_probes.remove(session_id); self.state = GateState::Probing { session_id: session_id.to_string(), since: now, @@ -265,6 +289,7 @@ impl NetworkGate { self.state = GateState::Open; self.probe_delay_ms = BASE_PROBE_DELAY_MS; self.probe_failures.clear(); + self.timed_out_probes.clear(); // The fingerprint is not read while the gate is open, so the // next closed period starts from a fresh baseline. self.addresses = None; @@ -284,9 +309,19 @@ impl NetworkGate { /// The probe has run for [`PROBE_TIMEOUT_MS`] without reaching its /// provider. The gate stops waiting for it: the schedule moves on as /// after a failed probe, and the session, still running, is left alone. + /// It is remembered, so that a network failure it reports later is + /// charged as the failed probe it is. fn probe_timed_out(&mut self, session_id: &str, now: i64) { - if self.is_probing(session_id) { - self.close_after_failed_probe(now); + if let GateState::Probing { + session_id: probe, + slept, + .. + } = &self.state + { + if probe == session_id { + self.timed_out_probes.insert(session_id.to_string(), *slept); + self.close_after_failed_probe(now); + } } } @@ -303,7 +338,31 @@ impl NetworkGate { /// Record that `session_id` failed on the network, and answer how the /// failure is charged to that session's retry budget. + /// + /// A session whose probe timed out earlier fails here as that probe, + /// late. Unless the gate has opened since, the failure is charged to the + /// session's run of failed probes under the same conditions as a probe + /// that failed in time. A closed gate also waits at least the probe + /// delay from now: the session's row is due at once, and with the timeout + /// long past it would otherwise probe again on the next tick. The delay + /// is not doubled a second time, because the timeout already doubled it + /// for this probe. fn record_failure(&mut self, session_id: &str, now: i64) -> PauseAccounting { + if let Some(slept) = self.timed_out_probes.remove(session_id) { + if self.state != GateState::Open { + if self.is_closed() { + self.next_probe_at = self + .next_probe_at + .max(now.saturating_add(self.probe_delay_ms)); + } + let charged = !slept && self.network_present(); + return if charged && self.probe_failures_exhausted(session_id) { + PauseAccounting::ProbesExhausted + } else { + PauseAccounting::Waiting + }; + } + } match &self.state { GateState::Open => { self.state = GateState::Closed; diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs index 655226d08..952607d38 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs @@ -885,3 +885,187 @@ fn a_timed_out_probe_closes_the_gate_like_a_failed_one_but_is_not_counted() { assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); assert!(gate.probe_failures.is_empty()); } + +/// The gate stopped waiting for `session_id`'s probe, and only then did the +/// session fail on the network, at `failed_at`. +fn fail_a_probe_late(gate: &mut NetworkGate, session_id: &str, failed_at: i64) -> PauseAccounting { + let started_at = failed_at - PROBE_TIMEOUT_MS - HOUR_MS; + gate.probe_started(session_id, started_at); + gate.probe_timed_out(session_id, started_at + PROBE_TIMEOUT_MS); + gate.record_failure(session_id, failed_at) +} + +/// A session whose probes always fail after the probe timeout is bounded +/// like one whose probes fail in time. +#[test] +fn a_timed_out_probe_that_fails_late_is_charged_to_the_sessions_probe_failures() { + let mut gate = gate_with_a_network(); + let mut now = 10 * HOUR_MS; + for failure in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!( + fail_a_probe_late(&mut gate, "lone", now), + PauseAccounting::Waiting, + "failure {failure}" + ); + assert_eq!(gate.probe_failures["lone"], failure); + now += 2 * HOUR_MS; + } + assert_eq!( + fail_a_probe_late(&mut gate, "lone", now), + PauseAccounting::ProbesExhausted + ); + assert!(gate.timed_out_probes.is_empty()); +} + +/// The late failure finds the gate closed with its probe time long past. +/// The next probe still waits the probe delay, which is not doubled again. +#[test] +fn a_late_probe_failure_does_not_start_the_next_probe_at_once() { + let mut gate = gate_with_a_network(); + gate.probe_started("lone", 1_000); + let timed_out_at = 1_000 + PROBE_TIMEOUT_MS; + gate.probe_timed_out("lone", timed_out_at); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + + let failed_at = timed_out_at + HOUR_MS; + assert_eq!(gate.turn(failed_at), NetworkTurn::Probe); + assert_eq!( + gate.record_failure("lone", failed_at), + PauseAccounting::Waiting + ); + + assert_eq!(gate.turn(failed_at), NetworkTurn::Hold); + assert_eq!(gate.next_probe_at, failed_at + 2 * BASE_PROBE_DELAY_MS); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + assert_eq!(gate.turn(gate.next_probe_at), NetworkTurn::Probe); +} + +/// A late failure never brings the next probe forward. +#[test] +fn a_late_probe_failure_keeps_a_later_probe_time() { + let mut gate = gate_with_a_network(); + gate.probe_started("lone", 1_000); + let timed_out_at = 1_000 + PROBE_TIMEOUT_MS; + gate.probe_timed_out("lone", timed_out_at); + gate.probe_delay_ms = 1_000; + + gate.record_failure("lone", timed_out_at + 1); + assert_eq!(gate.next_probe_at, timed_out_at + BASE_PROBE_DELAY_MS); +} + +#[test] +fn a_late_probe_failure_counts_only_awake_with_a_network_present() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + + // The machine slept after the gate stopped waiting for the probe. + gate.observe_tick(1_000); + gate.probe_started("lone", 1_000); + gate.probe_timed_out("lone", 1_000 + PROBE_TIMEOUT_MS); + gate.observe_tick(1_000 + PROBE_TIMEOUT_MS); + assert!(gate.observe_tick(1_000 + PROBE_TIMEOUT_MS + WAKE_GAP_MS + 1)); + assert_eq!( + gate.record_failure("lone", 2 * HOUR_MS), + PauseAccounting::Waiting + ); + + // It slept while the probe ran, before the timeout. + gate.observe_tick(3 * HOUR_MS); + gate.probe_started("lone", 3 * HOUR_MS); + assert!(gate.observe_tick(3 * HOUR_MS + PROBE_TIMEOUT_MS)); + gate.probe_timed_out("lone", 3 * HOUR_MS + PROBE_TIMEOUT_MS); + assert_eq!( + gate.record_failure("lone", 4 * HOUR_MS), + PauseAccounting::Waiting + ); + + // No interface holds an address. + assert!(gate.observe_fingerprint(BTreeSet::new())); + assert_eq!( + fail_a_probe_late(&mut gate, "lone", 6 * HOUR_MS), + PauseAccounting::Waiting + ); + + // None of those added to the run: the next late failure awake with a + // network is the twentieth. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe_late(&mut gate, "lone", 8 * HOUR_MS), + PauseAccounting::ProbesExhausted + ); +} + +/// While another session probes, the late failure is charged and the +/// running probe is left alone. +#[test] +fn a_late_probe_failure_during_another_probe_is_charged_and_leaves_it_alone() { + let mut gate = gate_with_a_network(); + gate.probe_started("hung", 1_000); + gate.probe_timed_out("hung", 1_000 + PROBE_TIMEOUT_MS); + let probing_at = gate.next_probe_at; + gate.probe_started("other", probing_at); + + assert_eq!( + gate.record_failure("hung", probing_at + 1_000), + PauseAccounting::Waiting + ); + assert_eq!(gate.probe_failures["hung"], 1); + assert!(gate.is_probing("other")); +} + +/// Once a probe gets through, an earlier timed-out probe is forgotten: its +/// failure with the gate open is counted like any other. +#[test] +fn a_probe_that_gets_through_forgets_the_timed_out_probes() { + let mut gate = gate_with_a_network(); + gate.probe_started("hung", 1_000); + gate.probe_timed_out("hung", 1_000 + PROBE_TIMEOUT_MS); + gate.probe_started("other", gate.next_probe_at); + gate.probe_succeeded("other"); + assert!(gate.timed_out_probes.is_empty()); + + assert_eq!( + gate.record_failure("hung", 2 * HOUR_MS), + PauseAccounting::Counted + ); + assert!(gate.probe_failures.is_empty()); +} + +/// A lone session whose probe fails after the timeout, through the +/// scheduler: the failure is charged, and the row, due at once, is not +/// probed again until the probe delay has passed. +#[test] +fn a_lone_session_that_fails_after_the_probe_timeout_waits_the_probe_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + scheduler + .network_gate + .lock() + .unwrap() + .observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!(tick(&scheduler, &store, now), sessions); + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS).is_empty()); + assert_eq!(scheduler.network_gate.lock().unwrap().probe(), None); + + let failed_at = now + PROBE_TIMEOUT_MS + HOUR_MS; + claude_turn_fails_offline(&store, &sessions[0]); + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.next_retry_at, Some(failed_at)); + assert_eq!( + scheduler.network_gate.lock().unwrap().probe_failures[&sessions[0]], + 1 + ); + assert!(tick(&scheduler, &store, failed_at).is_empty()); + assert!(tick(&scheduler, &store, failed_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, failed_at + 2 * BASE_PROBE_DELAY_MS), + sessions + ); +}