diff --git a/apps/staged/src-tauri/Cargo.lock b/apps/staged/src-tauri/Cargo.lock index d81845a0d..2ca5bb72c 100644 --- a/apps/staged/src-tauri/Cargo.lock +++ b/apps/staged/src-tauri/Cargo.lock @@ -20,6 +20,7 @@ dependencies = [ "flate2", "git-diff", "hex", + "if-addrs", "image", "imagesize", "include_dir", @@ -2630,6 +2631,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "image" version = "0.25.10" diff --git a/apps/staged/src-tauri/Cargo.toml b/apps/staged/src-tauri/Cargo.toml index 691abe855..569ec008d 100644 --- a/apps/staged/src-tauri/Cargo.toml +++ b/apps/staged/src-tauri/Cargo.toml @@ -21,6 +21,9 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } log = "0.4" libc = "0.2" +# Network interface addresses, read by the rate-limit scheduler's network +# gate to notice a network change while sessions are paused on the network. +if-addrs = "0.15" tauri = { version = "2.12.1", features = [] } tauri-plugin-log = "2.10.0" diff --git a/apps/staged/src-tauri/src/pipeline_dependency.rs b/apps/staged/src-tauri/src/pipeline_dependency.rs index dc1a2bf2b..f9aad6e05 100644 --- a/apps/staged/src-tauri/src/pipeline_dependency.rs +++ b/apps/staged/src-tauri/src/pipeline_dependency.rs @@ -68,8 +68,9 @@ fn status_gate(dependency: Option<&store::Session>) -> Option { let noun = dependency_noun(dependency); match dependency.status { // A disconnected dependency may still be running on its workstation, - // and a paused one hit a rate limit and resumes on its own, so neither - // has finished yet. + // and a paused one resumes on its own, whether it hit a provider + // limit or, with `rate_limit_kind = "network"`, its prompt failed on + // the network. Neither has finished yet. SessionStatus::Queued | SessionStatus::Running | SessionStatus::Disconnected diff --git a/apps/staged/src-tauri/src/project_mcp.rs b/apps/staged/src-tauri/src/project_mcp.rs index bdb0eb8d0..0bcd5dab5 100644 --- a/apps/staged/src-tauri/src/project_mcp.rs +++ b/apps/staged/src-tauri/src/project_mcp.rs @@ -827,7 +827,7 @@ impl ProjectToolsHandler { } #[tool( - description = "Wait for a repo session started by `start_repo_session`. Returns the current state and any available artifacts for the opaque `repo_session_id`; note content references attached images and videos by local file path. `wait_for_completion_seconds` defaults to 240. Calls without an MCP progress token return within 30 seconds; repeat the wait if the session is still queued or running, or disconnected (a remote session Staged lost its connection to; it may still be working). A `failed` session reports `completion_reason` and `error_message`; `remote_agent_lost` means its remote workspace agent died under it (the message names why, such as the workstation restarting), so work it left may be uncommitted on the workspace. A completed session has finished its one goal: follow up with `continue_repo_session` only to answer a question it ended on or to fix that outcome, and start a new repo session for whatever comes next." + description = "Wait for a repo session started by `start_repo_session`. Returns the current state and any available artifacts for the opaque `repo_session_id`; note content references attached images and videos by local file path. `wait_for_completion_seconds` defaults to 240. Calls without an MCP progress token return within 30 seconds; repeat the wait if the session is still queued or running, disconnected (a remote session Staged lost its connection to; it may still be working), or paused (its agent hit a provider limit or lost its network connection; Staged resumes a paused session by itself, so it needs no follow-up). A `failed` session reports `completion_reason` and `error_message`; `remote_agent_lost` means its remote workspace agent died under it (the message names why, such as the workstation restarting), so work it left may be uncommitted on the workspace. A completed session has finished its one goal: follow up with `continue_repo_session` only to answer a question it ended on or to fix that outcome, and start a new repo session for whatever comes next." )] async fn wait_for_repo_session( &self, @@ -946,11 +946,15 @@ impl ProjectToolsHandler { ); } } - // A session parked on a rate limit has no runner and no - // token: the store write is the whole cancel. It drops - // `completion_reason = rate_limited`, so the rate-limit - // scheduler never resumes the child, and settles the - // work queued behind it, as a Stop in the UI does. + // A paused session is parked for automatic resume, on a + // provider limit or, with `rate_limit_kind = "network"`, + // on a prompt that failed on the network. Either kind has + // no runner and no token: the store write is the whole + // cancel. It drops `completion_reason = rate_limited`, + // which both kinds carry, so the rate-limit scheduler + // never resumes the child and a network pause never + // probes with it, and settles the work queued behind it, + // as a Stop in the UI does. Ok(Some(session)) if session.status == SessionStatus::Paused => { crate::session_commands::cancel_session_without_runner( &self.store, @@ -1782,9 +1786,10 @@ mod tests { ); } - /// A child parked on a rate limit has no runner, so neither the queued - /// transition nor the registry can cancel it; the tool used to fall - /// through, report `paused`, and leave the scheduler to restart the child. + /// A paused child, parked on a provider limit or a network failure, has + /// no runner, so neither the queued transition nor the registry can cancel + /// it; the tool used to fall through, report `paused`, and leave the + /// scheduler to restart the child. /// The store write itself is covered by /// `session_commands::tests::cancelling_a_paused_session_without_a_runner_keeps_the_callers_reason_and_stops_auto_resume`; /// that the tool reaches it is pinned by source text, because the handler @@ -1824,6 +1829,20 @@ mod tests { .expect("cancel tool description"); assert!(wait_description.contains("Returns the current state")); + // A paused child is not finished and Staged restarts it, so the + // parent must be told to wait again, for either cause of the pause. + let repeat = &wait_description[wait_description + .find("repeat the wait if the session is still") + .expect("repeat-the-wait clause")..]; + let repeat = &repeat[..repeat.find("A `failed` session").expect("failed clause")]; + for state in ["queued", "running", "disconnected", "paused"] { + assert!( + repeat.contains(state), + "the wait description must tell the parent to wait again for `{state}`: {repeat}" + ); + } + assert!(repeat.contains("hit a provider limit or lost its network connection")); + assert!(repeat.contains("Staged resumes a paused session by itself")); assert!(!wait_description.contains("`activity` progress fields")); assert!(!wait_description.contains("`last_activity_at`")); assert!(!wait_description.contains("`last_tool_result`")); diff --git a/apps/staged/src-tauri/src/rate_limit_metrics.rs b/apps/staged/src-tauri/src/rate_limit_metrics.rs index f54a8ffa2..343115847 100644 --- a/apps/staged/src-tauri/src/rate_limit_metrics.rs +++ b/apps/staged/src-tauri/src/rate_limit_metrics.rs @@ -88,6 +88,26 @@ pub(crate) fn record_rate_limit_hit( ); } +/// A session was paused because its prompt failed on the network. Counted +/// apart from rate-limit hits; `signal` is what identified the failure. +pub(crate) fn record_network_pause(session_id: &str, provider: Option<&str>, signal: &str) { + let provider = metric_label(provider); + let snapshot = with_state(|state| { + state.increment_counter("network_pauses_total"); + state.snapshot.clone() + }); + + log_metric( + "network_pause", + json!({ + "sessionId": session_id, + "provider": provider, + "signal": signal, + }), + snapshot, + ); +} + /// Observe the count of paused sessions the scheduler can still act on. /// /// Sessions that gave up or need a manual resume are excluded by the caller, diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler.rs b/apps/staged/src-tauri/src/rate_limit_scheduler.rs index 98bd0474b..29c146a6b 100644 --- a/apps/staged/src-tauri/src/rate_limit_scheduler.rs +++ b/apps/staged/src-tauri/src/rate_limit_scheduler.rs @@ -1,4 +1,10 @@ -//! Jittered auto-recovery for sessions paused by provider API limits. +//! Jittered auto-recovery for sessions paused by provider API limits, and for +//! sessions whose prompt failed on the network ([`NETWORK_PAUSE_KIND`]), which +//! share the same pause. +//! +//! Network pauses do not back off one by one: they wait behind the shared gate +//! in [`network_gate`], which resumes one of them as a probe and the rest once +//! the probe gets through. //! //! Retry bookkeeping is stored under `provider_metadata.staged` so it survives //! restarts without widening the stable session schema. The scheduler only @@ -17,7 +23,59 @@ use tauri::AppHandle; use crate::actions::{ActionExecutor, ActionRegistry}; use crate::store::{MessageRole, Session, Store}; +mod network_gate; + +use network_gate::{InterfaceAddresses, NetworkFingerprint, NetworkGate, NetworkTurn}; + pub(crate) const AUTO_RETRY_RESUME_PROMPT: &str = "Continue where you left off."; +/// The resume prompt for a session paused by a network failure. The agent's +/// turn was cut off mid-request, so it is told to check its work first. +pub(crate) const NETWORK_RESUME_PROMPT: &str = "Your previous turn was cut off by a network error \ + before it finished. Check the state of your work, then continue where you left off."; + +/// The `rate_limit_kind` of a session paused because its prompt failed on the +/// network, not on a provider limit. +/// +/// Such a session shares the rate-limit pause (`paused` with +/// `CompletionReason::RateLimited`) so that Stop, the scheduler's claim and +/// the queue gate treat it the same way; this kind is the only thing that +/// tells the two apart. It carries no reset time. +pub(crate) const NETWORK_PAUSE_KIND: &str = "network"; + +/// The prompt an automatic resume sends when the user queued no message of +/// their own, chosen by what paused the session. +pub(crate) fn auto_resume_prompt(rate_limit_kind: Option<&str>) -> &'static str { + if rate_limit_kind == Some(NETWORK_PAUSE_KIND) { + NETWORK_RESUME_PROMPT + } else { + AUTO_RETRY_RESUME_PROMPT + } +} + +/// The bookkeeping `reason` of a network pause that waits for the gate +/// without spending a retry attempt. The app reads it to leave the retry +/// time out of its notice (`NETWORK_WAIT_REASON` in `sessionFailureCopy.ts`): +/// such a row's `next_retry_at` is the time of the pause, not of a retry. +const NETWORK_WAIT_REASON: &str = "network_wait"; +/// The bookkeeping `reason` of a network pause given up on because the +/// session's own probes kept failing with a network present. +const PROBE_LIMIT_REASON: &str = "probe_limit_exceeded"; + +/// How one pause is charged to its session's retry budget. +/// +/// Only a network pause is ever anything but `Counted`; the network gate +/// decides which (see `network_gate`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PauseAccounting { + /// Advances `attempt` under the attempt cap and the 24 hour window. + Counted, + /// Waits for the network gate: `attempt` stays, and neither the cap nor + /// the window applies. + Waiting, + /// The session's own probes failed too many times in a row with a + /// network present. It stays paused for a manual resume. + ProbesExhausted, +} const TICK_INTERVAL_SECS: u64 = 15; const MAX_AUTO_RETRY_ATTEMPTS: u32 = 8; @@ -38,15 +96,35 @@ const MAX_RESUMES_PER_TICK: usize = 2; /// to plain backoff instead. pub(crate) const MAX_PLAUSIBLE_RESET_LEAD_MS: i64 = 48 * 60 * 60 * 1_000; -#[derive(Default)] pub struct RateLimitScheduler { in_flight: Mutex>, tick_loop_spawned: AtomicBool, + /// The gate every network-paused session waits behind. + network_gate: Mutex, + /// Where the gate reads the machine's interface addresses from. + network_fingerprint: Box, +} + +impl Default for RateLimitScheduler { + fn default() -> Self { + Self::new() + } } impl RateLimitScheduler { pub fn new() -> Self { - Self::default() + Self::with_network_fingerprint(Box::new(InterfaceAddresses)) + } + + pub(crate) fn with_network_fingerprint( + network_fingerprint: Box, + ) -> Self { + Self { + in_flight: Mutex::default(), + tick_loop_spawned: AtomicBool::default(), + network_gate: Mutex::default(), + network_fingerprint, + } } fn try_mark_in_flight(&self, session_id: &str) -> bool { @@ -136,6 +214,8 @@ async fn run_tick( let now = crate::store::now_timestamp(); let mut started = 0usize; let mut drained_branches: HashSet = HashSet::new(); + scheduler.settle_network_probe(&store, now); + scheduler.observe_network_tick(now, network_rows_waiting(&sessions)); for session in sessions { if started >= MAX_RESUMES_PER_TICK { @@ -145,26 +225,11 @@ async fn run_tick( continue; } - let state = match ensure_auto_recovery_state(&store, &session, now) { - Ok(Some(state)) => state, - Ok(None) => { - scheduler.clear_in_flight(&session.id); - continue; - } - Err(e) => { - log::warn!( - "[rate_limit_recovery] failed to prepare session {}: {e}", - session.id - ); - scheduler.clear_in_flight(&session.id); - continue; - } - }; - - if !is_due(&state, now) { + let Some(admission) = admit_for_resume(&scheduler, &store, &session, now) else { scheduler.clear_in_flight(&session.id); continue; - } + }; + let state = admission.state; // The claim flips the row to `running` under our pid, and the runner // only registers it once the spawned task reaches `start_session` @@ -210,6 +275,11 @@ async fn run_tick( } }; + // Only now is the probe real: a deferred or lost claim above leaves + // the gate closed, so the next network row on this tick may probe. + if admission.probe { + scheduler.network_probe_started(&session.id, now); + } started += 1; crate::rate_limit_metrics::record_recovery_retry_attempt( &session.id, @@ -264,6 +334,7 @@ async fn run_tick( ); reschedule_after_start_failure( &store_for_task, + &scheduler_for_task, &session_id, crate::store::now_timestamp(), ); @@ -301,6 +372,61 @@ fn spawn_queue_drain(app_handle: &AppHandle, branch_id: &str) { ); } +/// A paused session the scheduler may claim on this tick. +struct ResumeAdmission { + state: RateLimitAutoRecovery, + /// The session resumes as the network gate's probe. + probe: bool, +} + +/// Whether `session` may be claimed on this tick: its own retry state is +/// due, and, for a network pause, the shared gate lets it through. +/// +/// API-limit pauses never consult the gate. +fn admit_for_resume( + scheduler: &RateLimitScheduler, + store: &Store, + session: &Session, + now: i64, +) -> Option { + let state = match ensure_auto_recovery_state(store, session, now) { + Ok(state) => state?, + Err(e) => { + log::warn!( + "[rate_limit_recovery] failed to prepare session {}: {e}", + session.id + ); + return None; + } + }; + if !is_due(&state, now) { + return None; + } + let probe = if is_network_pause(session.rate_limit_kind.as_deref()) { + match scheduler.network_turn(now) { + NetworkTurn::Hold => return None, + NetworkTurn::Probe => true, + NetworkTurn::Resume => false, + } + } else { + false + }; + Some(ResumeAdmission { state, probe }) +} + +/// Whether any listed session is paused on the network and may still resume +/// by itself: the rows the network gate could probe with. +fn network_rows_waiting(sessions: &[Session]) -> bool { + sessions.iter().any(|session| { + is_network_pause(session.rate_limit_kind.as_deref()) + && is_actionable_for_auto_resume(session) + }) +} + +fn is_network_pause(rate_limit_kind: Option<&str>) -> bool { + rate_limit_kind == Some(NETWORK_PAUSE_KIND) +} + fn is_due(state: &RateLimitAutoRecovery, now: i64) -> bool { if state.unsafe_to_auto_resume || state.gave_up_at.is_some() { return false; @@ -320,7 +446,9 @@ fn ensure_auto_recovery_state( return Ok(Some(state)); } - record_rate_limit_pause_for_session(store, session, now)?; + // No failure was reported for this row, so a network pause among these + // is not counted: it waits for the gate like the rest. + record_rate_limit_pause_for_session(store, session, PauseAccounting::Waiting, now)?; Ok(store .get_session(&session.id) .map_err(|e| e.to_string())? @@ -341,11 +469,19 @@ fn ensure_auto_recovery_state( /// row's column (not yet written) and from the row as it stands, which is all /// the blocker scan needs: `agent_id` and the message history, not status. /// +/// A network pause (`kind` of [`NETWORK_PAUSE_KIND`]) is reported to the +/// scheduler's network gate, which answers how it is charged to the +/// session's retry budget ([`PauseAccounting`]): only a failure while the +/// gate is open is counted. The gate hears of the failure even if a Stop +/// then wins the row, because the network failed either way. With no +/// scheduler the pause is not counted, as under a fresh gate. +/// /// Returns whether this call won the `running → paused` transition. A lost /// race (a Stop cancelled the row first) writes nothing, including the /// fragment. pub(crate) fn pause_for_rate_limit( store: &Store, + scheduler: Option<&RateLimitScheduler>, session_id: &str, error_message: Option<&str>, resets_at: Option<&str>, @@ -355,7 +491,14 @@ pub(crate) fn pause_for_rate_limit( let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(false); }; - let state = plan_recovery_state(store, &session, resets_at, now)?; + let accounting = if is_network_pause(kind) { + scheduler.map_or(PauseAccounting::Waiting, |scheduler| { + scheduler.record_network_failure(store, session_id, now) + }) + } else { + PauseAccounting::Counted + }; + let state = plan_recovery_state(store, &session, kind, accounting, resets_at, now)?; let transitioned = store .transition_running_to_rate_limited_with_provider_metadata( session_id, @@ -376,7 +519,7 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< let Some(session) = store.get_session(session_id).map_err(|e| e.to_string())? else { return Ok(()); }; - record_rate_limit_pause_for_session(store, &session, now) + record_rate_limit_pause_for_session(store, &session, PauseAccounting::Counted, now) } /// Write retry bookkeeping onto a row that is already paused. @@ -387,9 +530,17 @@ fn record_rate_limit_pause(store: &Store, session_id: &str, now: i64) -> Result< fn record_rate_limit_pause_for_session( store: &Store, session: &Session, + accounting: PauseAccounting, now: i64, ) -> Result<(), String> { - let state = plan_recovery_state(store, session, session.rate_limit_resets_at.as_deref(), now)?; + let state = plan_recovery_state( + store, + session, + session.rate_limit_kind.as_deref(), + accounting, + session.rate_limit_resets_at.as_deref(), + now, + )?; store .merge_session_provider_metadata(&session.id, &metadata_for_state(&state)) .map_err(|e| e.to_string())?; @@ -398,10 +549,12 @@ fn record_rate_limit_pause_for_session( } /// The next retry state for `session`, advancing any bookkeeping it already -/// carries. +/// carries. `kind` and `accounting` are those of [`build_recovery_state`]. fn plan_recovery_state( store: &Store, session: &Session, + kind: Option<&str>, + accounting: PauseAccounting, resets_at: Option<&str>, now: i64, ) -> Result { @@ -410,6 +563,8 @@ fn plan_recovery_state( let jitter_ms = jitter_ms(jitter_bound_for(resets_at, now)); Ok(build_recovery_state( existing.as_ref(), + kind, + accounting, resets_at, now, blocker.as_deref(), @@ -473,13 +628,30 @@ fn log_recovery_state(session: &Session, kind: Option<&str>, state: &RateLimitAu /// the 24h window in [`build_recovery_state`] still bound repeated failures, /// so give-up is reserved for starts that keep failing rather than issued on /// the first one. -fn reschedule_after_start_failure(store: &Store, session_id: &str, now: i64) { +/// +/// A start failure is not evidence about the network: `prepare_resume` fails +/// on a missing workspace path, a disconnected session or a branch in +/// transfer. So a network pause that fails to start is counted like any +/// other, which keeps it under the attempt cap, and if it was the network +/// gate's probe the probe is abandoned: another session may probe at once +/// and the probe delay does not grow. +fn reschedule_after_start_failure( + store: &Store, + scheduler: &RateLimitScheduler, + session_id: &str, + now: i64, +) { match store.get_session(session_id) { // Only reschedule a session the failed attempt actually parked; a // session in any other state was claimed by something else (a manual // resume, a cancel, a delete) and is that path's to manage. Ok(Some(session)) if session.status.as_str() == "paused" => { - if let Err(e) = record_rate_limit_pause_for_session(store, &session, now) { + if is_network_pause(session.rate_limit_kind.as_deref()) { + scheduler.abandon_network_probe(session_id, now); + } + if let Err(e) = + record_rate_limit_pause_for_session(store, &session, PauseAccounting::Counted, now) + { log::warn!( "[rate_limit_recovery] failed to reschedule session {session_id} after a failed start: {e}" ); @@ -505,8 +677,30 @@ fn is_actionable_for_auto_resume(session: &Session) -> bool { .unwrap_or(true) } +/// The retry state after one more pause. +/// +/// `accounting` applies to a network pause (`kind` of +/// [`NETWORK_PAUSE_KIND`]) only; an API-limit pause is always counted. +/// +/// - [`PauseAccounting::Waiting`]: the failure came while the network gate +/// was not open. The row keeps its `attempt`, is due at once so the gate +/// alone decides when it resumes, and is exempt from the attempt cap and +/// the 24 hour window. +/// - [`PauseAccounting::ProbesExhausted`]: the row gives up and waits for a +/// manual resume. +/// - [`PauseAccounting::Counted`]: the rules below, with one difference. The +/// 24 hour window of a network pause is measured only across counted +/// failures, not from its first pause: `first_limited_at` is set again +/// when `attempt` becomes 1, and when the pause before this one was not +/// counted (see [`last_pause_was_uncounted`]). Time spent waiting behind a +/// closed gate (a laptop asleep for a weekend) therefore never ends a +/// session, whether the wait came before its first counted failure or +/// between two of them. Counted failures with no wait between them still +/// give up once they span more than 24 hours. fn build_recovery_state( existing: Option<&RateLimitAutoRecovery>, + kind: Option<&str>, + accounting: PauseAccounting, resets_at: Option<&str>, now: i64, blocker: Option<&str>, @@ -527,7 +721,38 @@ fn build_recovery_state( }; } + let network = is_network_pause(kind); + if network && accounting == PauseAccounting::ProbesExhausted { + return RateLimitAutoRecovery { + attempt: existing.map(|s| s.attempt).unwrap_or(0), + max_attempts: MAX_AUTO_RETRY_ATTEMPTS, + next_retry_at: None, + first_limited_at, + updated_at: now, + gave_up_at: Some(now), + unsafe_to_auto_resume: false, + reason: Some(PROBE_LIMIT_REASON.to_string()), + }; + } + if network && accounting == PauseAccounting::Waiting { + return RateLimitAutoRecovery { + attempt: existing.map(|s| s.attempt).unwrap_or(0), + max_attempts: MAX_AUTO_RETRY_ATTEMPTS, + next_retry_at: Some(now), + first_limited_at, + updated_at: now, + gave_up_at: None, + unsafe_to_auto_resume: false, + reason: Some(NETWORK_WAIT_REASON.to_string()), + }; + } + let attempt = existing.map(|s| s.attempt.saturating_add(1)).unwrap_or(1); + let first_limited_at = if network && (attempt == 1 || last_pause_was_uncounted(existing)) { + now + } else { + first_limited_at + }; if attempt > MAX_AUTO_RETRY_ATTEMPTS { return RateLimitAutoRecovery { attempt, @@ -581,6 +806,16 @@ fn build_recovery_state( } } +/// Whether the bookkeeping left by the pause before this one shows that it +/// was a network pause not charged to the retry budget: a wait behind the +/// gate, or the end of the session's probes. The `reason` is that of the +/// latest pause, so no field is needed to tell. +fn last_pause_was_uncounted(existing: Option<&RateLimitAutoRecovery>) -> bool { + existing + .and_then(|state| state.reason.as_deref()) + .is_some_and(|reason| reason == NETWORK_WAIT_REASON || reason == PROBE_LIMIT_REASON) +} + /// The outcome of trying to take ownership of a rate-limited session for one /// auto-resume attempt. enum AutoResumeClaim { @@ -918,7 +1153,15 @@ mod tests { let now = parse_rfc3339_millis("2026-06-24T15:00:00Z").unwrap(); let reset = "2026-06-24T15:45:00Z"; - let state = build_recovery_state(None, Some(reset), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(reset), + now, + None, + 12_000, + ); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("reset_at")); @@ -930,7 +1173,15 @@ mod tests { #[test] fn schedules_unknown_reset_with_capped_backoff_jitter() { - let state = build_recovery_state(None, None, 1_000_000, None, 4_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + None, + 1_000_000, + None, + 4_000, + ); assert_eq!(state.attempt, 1); assert_eq!(state.reason.as_deref(), Some("backoff")); @@ -950,7 +1201,15 @@ mod tests { reason: Some("backoff".to_string()), }; - let state = build_recovery_state(Some(&existing), None, 10_000, None, 5_000); + let state = build_recovery_state( + Some(&existing), + None, + PauseAccounting::Counted, + None, + 10_000, + None, + 5_000, + ); assert_eq!(state.attempt, MAX_AUTO_RETRY_ATTEMPTS + 1); assert_eq!(state.gave_up_at, Some(10_000)); @@ -960,8 +1219,15 @@ mod tests { #[test] fn marks_missing_agent_session_as_manual_only() { - let state = - build_recovery_state(None, None, 10_000, Some("missing_agent_session_id"), 5_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + None, + 10_000, + Some("missing_agent_session_id"), + 5_000, + ); assert!(state.unsafe_to_auto_resume); assert_eq!(state.reason.as_deref(), Some("missing_agent_session_id")); @@ -1365,6 +1631,7 @@ mod tests { assert!(pause_for_rate_limit( &store, + None, &session.id, Some("rate limit"), Some("2026-06-24T15:45:00Z"), @@ -1419,10 +1686,16 @@ mod tests { ) .unwrap(); - assert!( - !pause_for_rate_limit(&store, &session.id, Some("rate limit"), None, None, 10_000) - .unwrap() - ); + assert!(!pause_for_rate_limit( + &store, + None, + &session.id, + Some("rate limit"), + None, + None, + 10_000 + ) + .unwrap()); let row = store.get_session(&session.id).unwrap().unwrap(); assert_eq!(row.status, SessionStatus::Cancelled); @@ -1518,6 +1791,279 @@ mod tests { ); } + /// A running session parked the way the runner parks a network failure, + /// with no gate to report to. + fn pause_for_network(store: &Store, session_id: &str) { + assert!(pause_for_rate_limit( + store, + None, + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + 10_000, + ) + .unwrap()); + } + + fn claimed_ticket( + store: &Arc, + session_id: &str, + ) -> Box { + match claim_for_auto_resume(store, session_id).unwrap() { + AutoResumeClaim::Claimed(ticket) => ticket, + _ => panic!("expected the idle branch to yield a claim"), + } + } + + #[test] + fn a_network_pause_waits_for_the_gate_without_spending_an_attempt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + pause_for_network(&store, &session.id); + + let listed = store.get_rate_limited_paused_sessions().unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].rate_limit_kind.as_deref(), Some("network")); + let state = auto_recovery_from_metadata(listed[0].provider_metadata.as_ref()).unwrap(); + assert_eq!(state.attempt, 0); + assert_eq!(state.next_retry_at, Some(10_000)); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + assert!(is_due(&state, 10_000)); + } + + /// The accounting belongs to network pauses: an API-limit pause advances + /// its attempt whatever it is given. + #[test] + fn an_api_limit_pause_is_always_counted() { + for accounting in [ + PauseAccounting::Counted, + PauseAccounting::Waiting, + PauseAccounting::ProbesExhausted, + ] { + let state = build_recovery_state( + None, + Some("claude_pro"), + accounting, + None, + 10_000, + None, + 5_000, + ); + assert_eq!(state.attempt, 1); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("backoff")); + } + } + + const DAY_MS: i64 = 24 * 60 * 60 * 1_000; + + fn network_state( + existing: Option<&RateLimitAutoRecovery>, + accounting: PauseAccounting, + now: i64, + ) -> RateLimitAutoRecovery { + build_recovery_state( + existing, + Some(NETWORK_PAUSE_KIND), + accounting, + None, + now, + None, + 5_000, + ) + } + + /// A row that waited behind the gate for more than a day, then fails + /// while the gate is open: its first counted failure starts the 24 hour + /// window, so it is scheduled, not given up on. + #[test] + fn the_first_counted_network_failure_after_a_long_wait_does_not_give_up() { + let paused_at = 1_000_000; + let waiting = network_state(None, PauseAccounting::Waiting, paused_at); + assert_eq!(waiting.first_limited_at, paused_at); + + let failed_at = paused_at + 30 * 60 * 60 * 1_000; + let counted = network_state(Some(&waiting), PauseAccounting::Counted, failed_at); + + assert_eq!(counted.attempt, 1); + assert_eq!(counted.gave_up_at, None); + assert_eq!(counted.reason.as_deref(), Some("backoff")); + assert_eq!(counted.first_limited_at, failed_at); + assert!(counted.next_retry_at.unwrap() > failed_at); + } + + /// The window still ends a network pause: counted failures with no wait + /// between them give up once they span more than 24 hours. + #[test] + fn the_network_window_runs_across_counted_failures() { + let first_counted_at = 1_000_000; + let first = network_state(None, PauseAccounting::Counted, first_counted_at); + assert_eq!(first.first_limited_at, first_counted_at); + + let inside = network_state( + Some(&first), + PauseAccounting::Counted, + first_counted_at + DAY_MS, + ); + assert_eq!(inside.attempt, 2); + assert_eq!(inside.gave_up_at, None); + assert_eq!(inside.first_limited_at, first_counted_at); + + let outside_at = first_counted_at + DAY_MS + 1; + let outside = network_state(Some(&inside), PauseAccounting::Counted, outside_at); + assert_eq!(outside.attempt, 3); + assert_eq!(outside.gave_up_at, Some(outside_at)); + assert_eq!(outside.reason.as_deref(), Some("retry_window_elapsed")); + } + + /// The session that closed the gate when the laptop slept: counted once, + /// then waiting behind the gate for more than a day. Its next counted + /// failure, on a flaky connection after the wake, restarts the window + /// and is scheduled, not given up on. + #[test] + fn a_wait_between_two_counted_network_failures_restarts_the_window() { + let first_counted_at = 1_000_000; + let first = network_state(None, PauseAccounting::Counted, first_counted_at); + + let waited_at = first_counted_at + DAY_MS / 2; + let waiting = network_state(Some(&first), PauseAccounting::Waiting, waited_at); + assert_eq!(waiting.attempt, 1); + + let second_counted_at = first_counted_at + 30 * 60 * 60 * 1_000; + let second = network_state(Some(&waiting), PauseAccounting::Counted, second_counted_at); + assert_eq!(second.attempt, 2); + assert_eq!(second.gave_up_at, None); + assert_eq!(second.reason.as_deref(), Some("backoff")); + assert_eq!(second.first_limited_at, second_counted_at); + + // The restarted window runs from that failure. + let inside = network_state( + Some(&second), + PauseAccounting::Counted, + second_counted_at + DAY_MS, + ); + assert_eq!(inside.gave_up_at, None); + assert_eq!(inside.first_limited_at, second_counted_at); + let outside_at = second_counted_at + DAY_MS + 1; + let outside = network_state(Some(&second), PauseAccounting::Counted, outside_at); + assert_eq!(outside.gave_up_at, Some(outside_at)); + assert_eq!(outside.reason.as_deref(), Some("retry_window_elapsed")); + } + + /// The attempt cap is not restarted by a wait. + #[test] + fn a_wait_between_counted_network_failures_keeps_the_attempt_cap() { + let mut state = network_state(None, PauseAccounting::Counted, 1_000_000); + for round in 1..=i64::from(MAX_AUTO_RETRY_ATTEMPTS) { + let now = 1_000_000 + round * 60_000; + state = network_state(Some(&state), PauseAccounting::Waiting, now); + state = network_state(Some(&state), PauseAccounting::Counted, now + 1); + } + assert_eq!(state.attempt, MAX_AUTO_RETRY_ATTEMPTS + 1); + assert_eq!(state.reason.as_deref(), Some("retry_cap_exceeded")); + } + + /// An API-limit pause keeps measuring from its first pause. + #[test] + fn the_api_limit_window_still_runs_from_the_first_pause() { + let first = build_recovery_state( + None, + Some("claude_pro"), + PauseAccounting::Counted, + None, + 1_000_000, + None, + 5_000, + ); + let late = 1_000_000 + DAY_MS + 1; + let state = build_recovery_state( + Some(&first), + Some("claude_pro"), + PauseAccounting::Counted, + None, + late, + None, + 5_000, + ); + assert_eq!(state.gave_up_at, Some(late)); + assert_eq!(state.reason.as_deref(), Some("retry_window_elapsed")); + } + + #[test] + fn exhausted_probes_give_up_and_wait_for_a_manual_resume() { + let waiting = network_state(None, PauseAccounting::Waiting, 10_000); + let state = network_state(Some(&waiting), PauseAccounting::ProbesExhausted, 20_000); + + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, Some(20_000)); + assert_eq!(state.next_retry_at, None); + assert_eq!(state.reason.as_deref(), Some("probe_limit_exceeded")); + assert!(!is_due(&state, i64::MAX)); + } + + /// A network pause with an unanswered tool call is manual-only whether or + /// not it is counted. + #[test] + fn an_unsafe_network_pause_stays_manual() { + let state = build_recovery_state( + None, + Some(NETWORK_PAUSE_KIND), + PauseAccounting::Waiting, + None, + 10_000, + Some("pending_tool_call"), + 5_000, + ); + assert!(state.unsafe_to_auto_resume); + assert_eq!(state.next_retry_at, None); + } + + #[test] + fn auto_resume_of_a_network_pause_sends_the_network_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + pause_for_network(&store, &session.id); + + let ticket = claimed_ticket(&store, &session.id); + + assert_eq!(ticket.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(ticket.prompt(), NETWORK_RESUME_PROMPT); + assert!(NETWORK_RESUME_PROMPT + .starts_with("Your previous turn was cut off by a network error before it finished. ")); + } + + #[test] + fn a_queued_message_replaces_the_network_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, session) = branch_with_running_session(&store); + store + .add_queued_session_message(&session.id, "actually do this instead", &[], None) + .unwrap(); + pause_for_network(&store, &session.id); + + let ticket = claimed_ticket(&store, &session.id); + + assert_eq!(ticket.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(ticket.prompt(), "actually do this instead"); + } + + #[test] + fn auto_resume_of_an_api_limit_pause_keeps_the_plain_prompt() { + let store = Arc::new(Store::in_memory().unwrap()); + let (_branch, paused) = branch_with_paused_session(&store); + + assert_eq!( + claimed_ticket(&store, &paused.id).prompt(), + AUTO_RETRY_RESUME_PROMPT + ); + assert_eq!( + auto_resume_prompt(Some("five_hour")), + AUTO_RETRY_RESUME_PROMPT + ); + assert_eq!(auto_resume_prompt(None), AUTO_RETRY_RESUME_PROMPT); + } + #[test] fn preserves_only_staged_auto_recovery_metadata() { let metadata = json!({ @@ -1575,18 +2121,42 @@ mod tests { // millennia out). Waiting for it would park the session forever: the // 24h give-up is only evaluated when recovery state is rebuilt. let micros = (parse_rfc3339_millis("2026-06-24T15:45:00Z").unwrap() * 1_000).to_string(); - let state = build_recovery_state(None, Some(µs), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(µs), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("backoff")); assert_eq!(jitter_bound_for(Some(µs), now), MAX_BACKOFF_MS); // The cap is on lead time, not scale: just beyond the window falls // back, just within it is honored. let beyond_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS + 60_000).to_string(); - let state = build_recovery_state(None, Some(&beyond_window), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(&beyond_window), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("backoff")); let within_window = (now + MAX_PLAUSIBLE_RESET_LEAD_MS - 60_000).to_string(); - let state = build_recovery_state(None, Some(&within_window), now, None, 12_000); + let state = build_recovery_state( + None, + None, + PauseAccounting::Counted, + Some(&within_window), + now, + None, + 12_000, + ); assert_eq!(state.reason.as_deref(), Some("reset_at")); assert_eq!(jitter_bound_for(Some(&within_window), now), RESET_JITTER_MS); } @@ -1704,7 +2274,7 @@ mod tests { // The failed attempt re-parked the session as paused; one failure // burns an attempt, not the whole recovery. - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); let state = auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap(); @@ -1735,7 +2305,7 @@ mod tests { ) .unwrap(); - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); let state = auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap(); @@ -1750,7 +2320,7 @@ mod tests { store.create_session(&session).unwrap(); // Still running: a manual resume won the race after the failed start. - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let session = store.get_session(&session.id).unwrap().unwrap(); assert_eq!( @@ -1794,7 +2364,7 @@ mod tests { ticket.rate_limit_kind.as_deref(), ) .unwrap()); - reschedule_after_start_failure(&store, &session.id, 20_000); + reschedule_after_start_failure(&store, &RateLimitScheduler::new(), &session.id, 20_000); let stopped = store.get_session(&session.id).unwrap().unwrap(); assert_eq!(stopped.status, SessionStatus::Cancelled); diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs new file mode 100644 index 000000000..2d287c060 --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate.rs @@ -0,0 +1,618 @@ +//! The shared gate that every network-paused session waits behind. +//! +//! A session paused with kind [`NETWORK_PAUSE_KIND`](super::NETWORK_PAUSE_KIND) +//! failed because the agent could not reach its model provider. That is a +//! fact about the machine, not about the session, so all such sessions share +//! one gate instead of each running its own backoff. API-limit pauses never +//! pass through it. +//! +//! | State | Meaning | What the scheduler does | +//! | ------- | -------------------------------- | ------------------------------- | +//! | Closed | The network is presumed down | Starts one probe when it is due | +//! | Probing | One probe turn is running | Starts no other network resume | +//! | Open | A probe reached its provider | Resumes the rest, two per tick | +//! +//! The gate lives in memory, owned by [`RateLimitScheduler`] behind a mutex, +//! and starts closed with a probe due at once, so a relaunch probes on its +//! first tick. +//! +//! A probe succeeds when its turn reaches the model provider (see +//! [`probe_verdict`]). A probe still running after [`PROBE_TIMEOUT_MS`] with +//! no such sign is given up on as a probe: the gate closes and the schedule +//! moves on as after a failed probe, while that session keeps running. A +//! user's manual resume of a network-paused session is watched as a probe +//! too when the gate is closed, so its success opens the gate for the rest. +//! +//! Failures while the gate is not open are not counted: the session's +//! `attempt` does not advance and the 24 hour window is not applied, so a +//! laptop that sleeps for days gives up on nothing. Failures while it is open +//! are counted under the existing cap, and close it again. +//! +//! One bound applies to uncounted failures. A session whose own probes fail +//! [`MAX_CONSECUTIVE_PROBE_FAILURES`] times in a row, each while the machine +//! stayed awake and held a network address, is given up on and left paused +//! for a manual resume: with a network evidently present, its failure is not +//! one that waiting fixes (a dead local proxy, a captive portal). +//! +//! A probe that fails only after the gate stopped waiting for it is still +//! that session's failed probe. The gate remembers each timed-out probe until +//! the gate opens or the session probes again, charges its late network +//! failure to the same bound under the same conditions, and holds the next +//! probe back by the probe delay, so a session that always fails late cannot +//! be probed again and again without limit. +//! +//! Two things make a closed gate probe sooner than its doubling delay, both +//! derived inside the scheduler tick with no operating-system callbacks: +//! +//! - **Wake.** A gap of more than [`WAKE_GAP_MS`] between two ticks means the +//! process was suspended. The probe delay resets, so a probe is due at once. +//! - **Network change.** While the gate is closed, each tick reads the +//! machine's [`NetworkFingerprint`]. A change resets the probe delay. +//! +//! An empty fingerprint means the machine has no network, so neither trigger +//! fires while it is empty. It never blocks a probe: the doubling schedule +//! keeps running, because the provider may be local (a model server or a +//! gateway on this machine) and reachable with no network at all. +//! +//! A probe starts only on a tick whose preceding gap was normal. A short +//! maintenance wake during sleep cannot produce two ticks one interval apart, +//! so it never starts a probe; after the lid opens, the second tick does. +//! +//! Every method takes `now`; the gate never reads a clock of its own. + +use std::collections::{BTreeSet, HashMap}; +use std::net::IpAddr; + +use crate::store::{CompletionReason, Session, SessionStatus, Store}; + +use super::{is_network_pause, PauseAccounting, RateLimitScheduler}; + +mod fingerprint; + +pub(crate) use fingerprint::{InterfaceAddresses, NetworkFingerprint}; + +/// The delay before the first probe after the gate closes, and the start of +/// the doubling for probes that fail. +pub(super) const BASE_PROBE_DELAY_MS: i64 = 15_000; +pub(super) const MAX_PROBE_DELAY_MS: i64 = 10 * 60_000; +/// A gap between two ticks longer than this means the process was suspended: +/// three of the scheduler's 15 second ticks. +pub(super) const WAKE_GAP_MS: i64 = 45_000; +/// How long a probe may run without reaching its provider before the gate +/// stops waiting for it, so one hung turn cannot hold the other rows. +/// +/// It is not known to be longer than an agent's own request timeout: that +/// was never measured, and an agent that retries a request that timed out +/// takes longer still. So a probe may fail on the network after this timeout +/// has passed. Such a late failure is charged to the session as a failed +/// probe (see [`NetworkGate::record_failure`]). +pub(super) const PROBE_TIMEOUT_MS: i64 = 10 * 60_000; +/// How many times in a row one session's own probes may fail, each while the +/// machine stayed awake and held a network address, before the session is +/// given up on and left paused for a manual resume. +pub(super) const MAX_CONSECUTIVE_PROBE_FAILURES: u32 = 20; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum GateState { + Closed, + /// `session_id` was claimed as the probe at `since`. + Probing { + session_id: String, + since: i64, + /// The process was suspended while the probe ran, so its failure + /// says nothing about a network the machine was awake to use. + slept: bool, + }, + Open, +} + +/// What the gate lets a due network-paused session do on this tick. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum NetworkTurn { + /// Start nothing. + Hold, + /// Resume this session as the probe. + Probe, + /// Resume normally: the gate is open. + Resume, +} + +#[derive(Debug)] +pub(super) struct NetworkGate { + state: GateState, + /// When a closed gate may start its next probe. + next_probe_at: i64, + /// The wait that the next failed probe adds before the one after it. + probe_delay_ms: i64, + /// The wall-clock time of the last tick. + last_tick_at: Option, + /// Whether the gap before the last tick was normal. A probe starts only + /// when it was; the first tick of a process has no gap and counts. + settled: bool, + /// The fingerprint last read while the gate was closed. `None` until the + /// first read, after the gate opens, and when a read fails. + addresses: Option>, + /// Per session, how many of its own probes failed in a row while the + /// machine was awake with a network address. Cleared when the gate opens. + probe_failures: HashMap, + /// The sessions whose probe timed out and whose turn has not been heard + /// of since, each with whether the process was suspended during that + /// turn. Cleared when the gate opens; a session leaves when it fails on + /// the network or probes again. + timed_out_probes: HashMap, +} + +impl Default for NetworkGate { + fn default() -> Self { + Self { + state: GateState::Closed, + next_probe_at: 0, + probe_delay_ms: BASE_PROBE_DELAY_MS, + last_tick_at: None, + settled: true, + addresses: None, + probe_failures: HashMap::new(), + timed_out_probes: HashMap::new(), + } + } +} + +impl NetworkGate { + pub(super) fn turn(&self, now: i64) -> NetworkTurn { + match self.state { + GateState::Open => NetworkTurn::Resume, + GateState::Probing { .. } => NetworkTurn::Hold, + GateState::Closed if now >= self.next_probe_at && self.settled => NetworkTurn::Probe, + GateState::Closed => NetworkTurn::Hold, + } + } + + /// Record a scheduler tick, and answer whether the gap before it shows + /// that the process was suspended. The caller decides whether that + /// resets the probe delay (see [`Self::apply_triggers`]). + fn observe_tick(&mut self, now: i64) -> bool { + let woke = self + .last_tick_at + .is_some_and(|last| now.saturating_sub(last) > WAKE_GAP_MS); + self.last_tick_at = Some(now); + self.settled = !woke; + if woke { + if let GateState::Probing { slept, .. } = &mut self.state { + *slept = true; + } + for slept in self.timed_out_probes.values_mut() { + *slept = true; + } + } + woke + } + + /// Something suggests the network may be back. A closed gate may probe + /// at once, and the next failed probe waits the base delay again. A + /// running probe is left to finish. + fn reset_probe_delay(&mut self, now: i64) { + self.probe_delay_ms = BASE_PROBE_DELAY_MS; + if self.state == GateState::Closed { + self.next_probe_at = now; + } + } + + /// Act on this tick's triggers: a wake, or a changed fingerprint. Either + /// resets the probe delay, unless the fingerprint is empty: a machine + /// with no network has nothing new to probe, so the doubling schedule + /// stands. Answers whether the delay was reset. + fn apply_triggers(&mut self, woke: bool, changed: bool, now: i64) -> bool { + let fire = (woke || changed) && !self.network_absent(); + if fire { + self.reset_probe_delay(now); + } + fire + } + + fn is_closed(&self) -> bool { + self.state == GateState::Closed + } + + /// The last fingerprint read held no address: there is no network. + fn network_absent(&self) -> bool { + self.addresses.as_ref().is_some_and(BTreeSet::is_empty) + } + + /// The last fingerprint read held an address. `false` when it held none + /// and when nothing is known. + fn network_present(&self) -> bool { + self.addresses + .as_ref() + .is_some_and(|addresses| !addresses.is_empty()) + } + + /// Record the fingerprint read on this tick, and answer whether it + /// differs from the one before. The first read is the baseline, not a + /// change. + fn observe_fingerprint(&mut self, addresses: BTreeSet) -> bool { + if !self.is_closed() { + return false; + } + let changed = self + .addresses + .as_ref() + .is_some_and(|previous| *previous != addresses); + self.addresses = Some(addresses); + changed + } + + /// The fingerprint could not be read. Nothing is known about the + /// network, so an earlier "no network" answer no longer holds back the + /// triggers. + fn fingerprint_unreadable(&mut self) { + self.addresses = None; + } + + fn probe_started(&mut self, session_id: &str, now: i64) { + // A session that probes again was parked since its probe timed out, + // so that turn is over. + self.timed_out_probes.remove(session_id); + self.state = GateState::Probing { + session_id: session_id.to_string(), + since: now, + slept: false, + }; + } + + /// The user resumed a network-paused session by hand. A closed gate + /// watches that turn as its probe, so its answer opens the gate for the + /// rest; a gate already probing or open has its answer coming. + fn adopt_probe(&mut self, session_id: &str, now: i64) -> bool { + let adopt = self.is_closed(); + if adopt { + self.probe_started(session_id, now); + } + adopt + } + + fn probe(&self) -> Option<(String, i64)> { + match &self.state { + GateState::Probing { + session_id, since, .. + } => Some((session_id.clone(), *since)), + _ => None, + } + } + + fn is_probing(&self, session_id: &str) -> bool { + matches!(&self.state, GateState::Probing { session_id: probe, .. } if probe == session_id) + } + + /// The probe's turn reached its provider: the network is back. + fn probe_succeeded(&mut self, session_id: &str) { + if self.is_probing(session_id) { + self.state = GateState::Open; + self.probe_delay_ms = BASE_PROBE_DELAY_MS; + self.probe_failures.clear(); + self.timed_out_probes.clear(); + // The fingerprint is not read while the gate is open, so the + // next closed period starts from a fresh baseline. + self.addresses = None; + } + } + + /// The probe ended without saying anything about the network: it was + /// stopped, deleted, or failed for another reason. Another session may + /// probe at once, and the delay is left alone. + fn probe_abandoned(&mut self, session_id: &str, now: i64) { + if self.is_probing(session_id) { + self.state = GateState::Closed; + self.next_probe_at = now; + } + } + + /// The probe has run for [`PROBE_TIMEOUT_MS`] without reaching its + /// provider. The gate stops waiting for it: the schedule moves on as + /// after a failed probe, and the session, still running, is left alone. + /// It is remembered, so that a network failure it reports later is + /// charged as the failed probe it is. + fn probe_timed_out(&mut self, session_id: &str, now: i64) { + if let GateState::Probing { + session_id: probe, + slept, + .. + } = &self.state + { + if probe == session_id { + self.timed_out_probes.insert(session_id.to_string(), *slept); + self.close_after_failed_probe(now); + } + } + } + + /// Close the gate after a probe that did not get through: the next one + /// waits the current delay, and the one after it twice that. + fn close_after_failed_probe(&mut self, now: i64) { + self.state = GateState::Closed; + self.next_probe_at = now.saturating_add(self.probe_delay_ms); + self.probe_delay_ms = self + .probe_delay_ms + .saturating_mul(2) + .min(MAX_PROBE_DELAY_MS); + } + + /// Record that `session_id` failed on the network, and answer how the + /// failure is charged to that session's retry budget. + /// + /// A session whose probe timed out earlier fails here as that probe, + /// late. Unless the gate has opened since, the failure is charged to the + /// session's run of failed probes under the same conditions as a probe + /// that failed in time. A closed gate also waits at least the probe + /// delay from now: the session's row is due at once, and with the timeout + /// long past it would otherwise probe again on the next tick. The delay + /// is not doubled a second time, because the timeout already doubled it + /// for this probe. + fn record_failure(&mut self, session_id: &str, now: i64) -> PauseAccounting { + if let Some(slept) = self.timed_out_probes.remove(session_id) { + if self.state != GateState::Open { + if self.is_closed() { + self.next_probe_at = self + .next_probe_at + .max(now.saturating_add(self.probe_delay_ms)); + } + let charged = !slept && self.network_present(); + return if charged && self.probe_failures_exhausted(session_id) { + PauseAccounting::ProbesExhausted + } else { + PauseAccounting::Waiting + }; + } + } + match &self.state { + GateState::Open => { + self.state = GateState::Closed; + self.next_probe_at = now.saturating_add(BASE_PROBE_DELAY_MS); + self.probe_delay_ms = BASE_PROBE_DELAY_MS.saturating_mul(2); + PauseAccounting::Counted + } + GateState::Probing { + session_id: probe, + slept, + .. + } if probe == session_id => { + let awake = !*slept; + self.close_after_failed_probe(now); + if awake && self.network_present() && self.probe_failures_exhausted(session_id) { + PauseAccounting::ProbesExhausted + } else { + PauseAccounting::Waiting + } + } + // A session that was already running when the network dropped, + // failing while the gate is closed or another session probes. It + // says nothing new, so the probe schedule stands. + GateState::Probing { .. } | GateState::Closed => PauseAccounting::Waiting, + } + } + + /// Count one more failed probe of `session_id` made with the machine + /// awake and a network present, and answer whether that was its last. + fn probe_failures_exhausted(&mut self, session_id: &str) -> bool { + let failures = self + .probe_failures + .entry(session_id.to_string()) + .or_insert(0); + *failures += 1; + let exhausted = *failures >= MAX_CONSECUTIVE_PROBE_FAILURES; + if exhausted { + self.probe_failures.remove(session_id); + } + exhausted + } +} + +/// How a running probe stands, read from the store. +#[derive(Debug, PartialEq, Eq)] +enum ProbeVerdict { + Running, + Succeeded, + Abandoned, +} + +/// A probe succeeds when its turn shows, after the claim, that a request +/// reached the provider: a tool call or its result, a result the agent did +/// not mark as an error, or a prompt response (see +/// [`Store::session_reached_provider_since`]). Message text does not count: +/// a failed turn records its own error as text. A turn that completed did +/// get its prompt response, whether or not the row for it is in the store +/// yet. +fn probe_verdict(store: &Store, session_id: &str, since: i64) -> Result { + if store + .session_reached_provider_since(session_id, since) + .map_err(|e| e.to_string())? + { + return Ok(ProbeVerdict::Succeeded); + } + let status = store + .get_session(session_id) + .map_err(|e| e.to_string())? + .map(|session| session.status); + Ok(match status { + Some(SessionStatus::Running) => ProbeVerdict::Running, + Some(SessionStatus::Completed) => ProbeVerdict::Succeeded, + // Stopped, deleted, ended on another error, or parked by something + // other than this process's pause path (which reports a network + // failure to the gate before it writes the pause). + _ => ProbeVerdict::Abandoned, + }) +} + +impl RateLimitScheduler { + /// What the gate lets a due network-paused session do on this tick. + pub(super) fn network_turn(&self, now: i64) -> NetworkTurn { + self.network_gate.lock().unwrap().turn(now) + } + + /// Apply the wake and network-change triggers for the tick at `now`. + /// + /// Runs once per tick, before any row is admitted. The fingerprint is + /// read only while the gate is closed and `network_rows_waiting`, so a + /// machine with nothing paused on the network never lists its interfaces. + pub(super) fn observe_network_tick(&self, now: i64, network_rows_waiting: bool) { + let woke = { + let mut gate = self.network_gate.lock().unwrap(); + let woke = gate.observe_tick(now); + if !gate.is_closed() || !network_rows_waiting { + // No fingerprint is read, so nothing says the network is + // absent: a wake resets the delay. + if woke && gate.apply_triggers(true, false, now) { + log::info!("[network_gate] woke from a suspend; probe delay reset"); + } + return; + } + woke + }; + // The interfaces are read with the gate unlocked; the gate rechecks + // that it is still closed before it takes the answer. + let addresses = self.network_fingerprint.read(); + let mut gate = self.network_gate.lock().unwrap(); + let changed = match addresses { + Ok(addresses) => gate.observe_fingerprint(addresses), + Err(e) => { + log::warn!("[network_gate] failed to read network interfaces: {e}"); + gate.fingerprint_unreadable(); + false + } + }; + if gate.apply_triggers(woke, changed, now) { + log::info!( + "[network_gate] {}; probe delay reset", + if woke { + "woke from a suspend" + } else { + "network interfaces changed" + } + ); + } else if woke || changed { + log::info!( + "[network_gate] no network interface has an address; probe schedule unchanged" + ); + } + } + + /// The scheduler claimed `session_id` as the probe. + pub(super) fn network_probe_started(&self, session_id: &str, now: i64) { + log::info!("[network_gate] probing with session {session_id}"); + self.network_gate + .lock() + .unwrap() + .probe_started(session_id, now); + } + + /// The user resumed `session` by hand and the claim won. If it was paused + /// on the network and the gate is closed, the gate watches the turn as + /// its probe: without this a resume that gets through tells the gate + /// nothing, and the other paused sessions wait out the probe delay. + pub(crate) fn manual_resume_started(&self, session: &Session, now: i64) { + let network_paused = session.status == SessionStatus::Paused + && session.completion_reason == Some(CompletionReason::RateLimited) + && is_network_pause(session.rate_limit_kind.as_deref()); + if network_paused + && self + .network_gate + .lock() + .unwrap() + .adopt_probe(&session.id, now) + { + log::info!( + "[network_gate] watching manually resumed session {} as the probe", + session.id + ); + } + } + + /// The probe `session_id` ended without saying anything about the + /// network: its resume failed to start. Another session may probe at + /// once, and the probe delay is left alone. + pub(super) fn abandon_network_probe(&self, session_id: &str, now: i64) { + self.network_gate + .lock() + .unwrap() + .probe_abandoned(session_id, now); + } + + /// Move the gate out of `Probing` if the store shows how the probe went, + /// or if it has run past [`PROBE_TIMEOUT_MS`]. + /// + /// Runs at the start of each tick, and before a network failure is + /// recorded so that a probe which reached its provider and then failed + /// is counted as a failure with the gate open. + pub(super) fn settle_network_probe(&self, store: &Store, now: i64) { + self.settle_probe(store, None, now); + } + + /// [`Self::settle_network_probe`], with the session whose network failure + /// is about to be recorded. If that session is the probe, its failure is + /// what settles it: a row that no longer runs is not an abandoned probe, + /// and one that ran long is not timed out. + fn settle_probe(&self, store: &Store, failing: Option<&str>, now: i64) { + // The store is read with the gate unlocked; the gate rechecks that it + // is still probing this session before it moves. + let Some((session_id, since)) = self.network_gate.lock().unwrap().probe() else { + return; + }; + match probe_verdict(store, &session_id, since) { + Ok(ProbeVerdict::Running | ProbeVerdict::Abandoned) + if failing == Some(session_id.as_str()) => {} + Ok(ProbeVerdict::Running) if now.saturating_sub(since) >= PROBE_TIMEOUT_MS => { + log::warn!( + "[network_gate] probe session {session_id} has not reached its provider \ + after {} s; no longer waiting for it", + PROBE_TIMEOUT_MS / 1_000 + ); + self.network_gate + .lock() + .unwrap() + .probe_timed_out(&session_id, now); + } + Ok(ProbeVerdict::Running) => {} + Ok(ProbeVerdict::Succeeded) => { + log::info!("[network_gate] probe session {session_id} got through; gate open"); + self.network_gate + .lock() + .unwrap() + .probe_succeeded(&session_id); + } + Ok(ProbeVerdict::Abandoned) => { + log::info!("[network_gate] probe session {session_id} ended without an answer"); + self.network_gate + .lock() + .unwrap() + .probe_abandoned(&session_id, now); + } + Err(e) => { + log::warn!("[network_gate] failed to read probe session {session_id}: {e}"); + } + } + } + + /// Record that `session_id` failed on the network, and answer how the + /// failure is charged to that session's retry budget: counted only when + /// the gate was open, and the end of the session's automatic resumes when + /// its own probes have failed [`MAX_CONSECUTIVE_PROBE_FAILURES`] times. + pub(super) fn record_network_failure( + &self, + store: &Store, + session_id: &str, + now: i64, + ) -> PauseAccounting { + self.settle_probe(store, Some(session_id), now); + let accounting = self + .network_gate + .lock() + .unwrap() + .record_failure(session_id, now); + log::info!( + "[network_gate] session {session_id} failed on the network ({accounting:?}); gate closed" + ); + accounting + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs new file mode 100644 index 000000000..554f5fc50 --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/fingerprint.rs @@ -0,0 +1,36 @@ +//! The machine's network fingerprint: the set of addresses its interfaces +//! hold. The gate reads it while closed to notice that the network changed, +//! or that there is none. "None" only holds back the wake and change +//! triggers; it never stops a probe, since a provider on this machine or on +//! a link-local network needs none of these addresses. + +use std::collections::BTreeSet; +use std::io; +use std::net::IpAddr; + +/// Reads the set of interface addresses that could carry a request off this +/// machine. An empty set means there is no network. +pub(crate) trait NetworkFingerprint: Send + Sync { + fn read(&self) -> io::Result>; +} + +/// The real fingerprint, read from the operating system's interface list. +pub(crate) struct InterfaceAddresses; + +impl NetworkFingerprint for InterfaceAddresses { + fn read(&self) -> io::Result> { + Ok(if_addrs::get_if_addrs()? + .into_iter() + .filter(counts_as_network) + .map(|interface| interface.ip()) + .collect()) + } +} + +/// Loopback addresses never leave the machine. Link-local ones are left out +/// too: macOS keeps an `fe80::` address on its `utun` and `awdl` interfaces +/// with Wi-Fi off and no cable, so counting them would make the set never +/// empty and "no network" undetectable. +fn counts_as_network(interface: &if_addrs::Interface) -> bool { + !interface.is_loopback() && !interface.is_link_local() +} diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs new file mode 100644 index 000000000..952607d38 --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests.rs @@ -0,0 +1,1071 @@ +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use super::super::{ + admit_for_resume, auto_recovery_from_metadata, claim_for_auto_resume, network_rows_waiting, + pause_for_rate_limit, reschedule_after_start_failure, AutoResumeClaim, PauseAccounting, + RateLimitAutoRecovery, RateLimitScheduler, MAX_AUTO_RETRY_ATTEMPTS, MAX_RESUMES_PER_TICK, + NETWORK_PAUSE_KIND, +}; +use super::*; + +mod triggers; +use crate::store::{ + AcpMessageMetadata, CompletionReason, MessageRole, Session, SessionStatus, Store, +}; + +const HOUR_MS: i64 = 60 * 60 * 1_000; + +/// The store stamps rows with the wall clock, and the scheduler lists paused +/// rows by that stamp. A short sleep keeps two writes in order. +fn step() { + std::thread::sleep(Duration::from_millis(2)); +} + +fn running_session(store: &Store) -> Session { + let session = Session::new_running("work", Path::new("/tmp")).with_agent("agent-1"); + store.create_session(&session).unwrap(); + session +} + +/// Park a running session the way the runner parks a network failure. +fn fail_on_network(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + step(); + assert!(pause_for_rate_limit( + store, + Some(scheduler), + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + now, + ) + .unwrap()); +} + +fn network_paused_sessions( + store: &Store, + scheduler: &RateLimitScheduler, + count: usize, + now: i64, +) -> Vec { + (0..count) + .map(|_| { + let session = running_session(store); + fail_on_network(store, scheduler, &session.id, now); + session.id + }) + .collect() +} + +/// The synchronous part of `run_tick` without the wake and network-change +/// triggers: settle the probe, then admit and claim rows oldest pause first. +/// Returns the sessions it started. +/// +/// The tests in this file jump the clock freely to exercise the gate's own +/// rules; `triggers.rs` drives [`tick_with_triggers`] on a real cadence. +fn tick(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { + scheduler.settle_network_probe(store, now); + admit_and_claim(scheduler, store, now) +} + +/// The whole synchronous part of `run_tick`, triggers included. +fn tick_with_triggers(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { + let sessions = store.get_rate_limited_paused_sessions().unwrap(); + scheduler.settle_network_probe(store, now); + scheduler.observe_network_tick(now, network_rows_waiting(&sessions)); + admit_and_claim(scheduler, store, now) +} + +fn admit_and_claim(scheduler: &RateLimitScheduler, store: &Arc, now: i64) -> Vec { + let mut started = Vec::new(); + for session in store.get_rate_limited_paused_sessions().unwrap() { + if started.len() >= MAX_RESUMES_PER_TICK { + break; + } + let Some(admission) = admit_for_resume(scheduler, store, &session, now) else { + continue; + }; + if let AutoResumeClaim::Claimed(_) = claim_for_auto_resume(store, &session.id).unwrap() { + if admission.probe { + scheduler.network_probe_started(&session.id, now); + } + started.push(session.id); + } + } + started +} + +fn recovery(store: &Store, session_id: &str) -> RateLimitAutoRecovery { + let session = store.get_session(session_id).unwrap().unwrap(); + auto_recovery_from_metadata(session.provider_metadata.as_ref()).unwrap() +} + +/// The agent's turn reaches its provider: the model asks for a tool, and +/// the tool answers. +fn agent_replies(store: &Store, session_id: &str) { + for (role, content) in [ + (MessageRole::ToolCall, "Read"), + (MessageRole::ToolResult, "contents"), + ] { + store + .add_session_message(session_id, role, content) + .unwrap(); + } +} + +const OFFLINE_ERROR: &str = + "API Error: Can't reach the API server \u{2014} check your internet or DNS (ENOTFOUND)"; + +fn acp_event(store: &Store, session_id: &str, kind: &str, content: serde_json::Value) { + store + .add_acp_metadata_message( + session_id, + &AcpMessageMetadata { + acp_event_kind: Some(kind.to_string()), + acp_content: Some(content), + ..Default::default() + }, + ) + .unwrap(); +} + +/// The rows a Claude turn records when it cannot reach the provider, as +/// stored by real failed turns: the CLI's error as a message chunk, the +/// SDK's result marked as an error (its `subtype` still reads `success`), a +/// usage update, the prompt error, and the error text as the visible +/// assistant message. +fn claude_turn_fails_offline(store: &Store, session_id: &str) { + acp_event( + store, + session_id, + "agent_message_chunk", + serde_json::json!({ + "content": { "type": "text", "text": OFFLINE_ERROR }, + "messageId": "a6922e42-33a1-4c9b-95d6-45c748f7e168", + }), + ); + acp_event( + store, + session_id, + "response_result", + serde_json::json!({ + "type": "result", + "subtype": "success", + "is_error": true, + "terminal_reason": "api_error", + "api_error": "connection_failed", + "api_error_status": null, + "stop_reason": "stop_sequence", + "num_turns": 32, + "result": OFFLINE_ERROR, + }), + ); + acp_event( + store, + session_id, + "usage_update", + serde_json::json!({ "used": 175_506, "size": 1_000_000 }), + ); + acp_event( + store, + session_id, + "prompt_error", + serde_json::json!({ + "code": -32603, + "message": format!("Internal error: {OFFLINE_ERROR}"), + "data": { "errorKind": "server_error" }, + }), + ); + store + .add_session_message(session_id, MessageRole::Assistant, OFFLINE_ERROR) + .unwrap(); +} + +/// A gate opened by `sessions[0]` probing and getting an answer. +fn open_gate(scheduler: &RateLimitScheduler, store: &Arc, sessions: &[String], now: i64) { + assert_eq!(tick(scheduler, store, now), [sessions[0].clone()]); + agent_replies(store, &sessions[0]); + scheduler.settle_network_probe(store, now); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); +} + +#[test] +fn a_fresh_scheduler_starts_closed_with_an_immediate_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + assert_eq!(scheduler.network_turn(0), NetworkTurn::Probe); + + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // The first failure does not push the first probe out. + assert_eq!(tick(&scheduler, &store, now), sessions); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); +} + +#[test] +fn only_one_probe_starts_for_eleven_paused_rows() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 11, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + // While the probe runs, later ticks start nothing. + assert!(tick(&scheduler, &store, now + 15_000).is_empty()); + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS - 1).is_empty()); + assert_eq!(store.get_rate_limited_paused_sessions().unwrap().len(), 10); +} + +#[test] +fn a_failed_probe_spends_no_attempt_and_rotates_to_the_next_row() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 3, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + let failed_at = now + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.next_retry_at, Some(failed_at)); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + + // The next probe waits out the delay, then goes to the row that has been + // paused longest, which is no longer the one that just failed. + assert!(tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + let second_at = failed_at + BASE_PROBE_DELAY_MS; + assert_eq!(tick(&scheduler, &store, second_at), [sessions[1].clone()]); + + // A second failed probe doubles the delay. + fail_on_network(&store, &scheduler, &sessions[1], second_at); + assert!(tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS), + [sessions[2].clone()] + ); +} + +#[test] +fn a_probe_with_agent_output_opens_the_gate_and_the_rest_resume_two_per_tick() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + // Output from before the claim says nothing about the network now. + let first = running_session(&store); + agent_replies(&store, &first.id); + step(); + let now = crate::store::now_timestamp(); + fail_on_network(&store, &scheduler, &first.id, now); + let rest = network_paused_sessions(&store, &scheduler, 4, now); + + assert_eq!( + tick(&scheduler, &store, now), + std::slice::from_ref(&first.id) + ); + assert!(tick(&scheduler, &store, now).is_empty()); + + agent_replies(&store, &first.id); + assert_eq!(tick(&scheduler, &store, now), rest[..2]); + assert_eq!(tick(&scheduler, &store, now), rest[2..]); + for session_id in &rest { + assert_eq!(recovery(&store, session_id).attempt, 0); + } +} + +#[test] +fn a_probe_that_completes_opens_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + store + .update_session_status( + &sessions[0], + SessionStatus::Completed, + None, + Some(&CompletionReason::TurnComplete), + ) + .unwrap(); + + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); +} + +#[test] +fn a_stopped_probe_hands_the_probe_to_the_next_row_at_once() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + store + .update_session_status( + &sessions[0], + SessionStatus::Cancelled, + None, + Some(&CompletionReason::Interrupted), + ) + .unwrap(); + + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + // It is a probe, not a normal resume: the gate is still shut. + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); +} + +#[test] +fn a_failure_while_open_advances_the_attempt_and_closes_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 4, now); + open_gate(&scheduler, &store, &sessions, now); + assert_eq!(tick(&scheduler, &store, now), sessions[1..3]); + + let failed_at = now + 5_000; + fail_on_network(&store, &scheduler, &sessions[1], failed_at); + + let state = recovery(&store, &sessions[1]); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert!(state.next_retry_at.unwrap() > failed_at); + + // Closed: the row still waiting does not resume until a probe is due. + assert!(tick(&scheduler, &store, failed_at).is_empty()); + assert_eq!( + tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS), + [sessions[3].clone()] + ); + + // The other session resumed in the open period fails with the gate shut, + // so it is not counted. + fail_on_network(&store, &scheduler, &sessions[2], failed_at + 1_000); + assert_eq!(recovery(&store, &sessions[2]).attempt, 0); +} + +#[test] +fn a_probe_that_answers_and_then_fails_is_counted() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + assert_eq!(tick(&scheduler, &store, now), sessions); + agent_replies(&store, &sessions[0]); + // No tick has seen the output yet; the pause path settles the probe. + fail_on_network(&store, &scheduler, &sessions[0], now + 1_000); + + assert_eq!(recovery(&store, &sessions[0]).attempt, 1); + assert_eq!(scheduler.network_turn(now + 1_000), NetworkTurn::Hold); +} + +#[test] +fn a_row_past_24_hours_still_resumes() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // Two failed probes a day apart: neither is counted, and neither gives up. + let mut at = now; + for _ in 0..2 { + at += 25 * HOUR_MS; + assert_eq!(tick(&scheduler, &store, at), sessions); + fail_on_network(&store, &scheduler, &sessions[0], at); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.first_limited_at, now); + } + assert_eq!(tick(&scheduler, &store, at + 25 * HOUR_MS), sessions); +} + +#[test] +fn api_limit_rows_are_unaffected_by_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let network = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [network[0].clone()]); + + // An API limit while a probe runs: counted as ever, and no word to the + // gate. + let limited = running_session(&store); + step(); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + now, + ) + .unwrap()); + let state = recovery(&store, &limited.id); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); + + // It resumes on its own schedule, past the network row the gate holds. + let due = state.next_retry_at.unwrap(); + assert!(tick(&scheduler, &store, due - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, due), + std::slice::from_ref(&limited.id) + ); + assert_eq!(scheduler.network_turn(due), NetworkTurn::Hold); + + // Nor does an API limit close an open gate. + agent_replies(&store, &network[0]); + scheduler.settle_network_probe(&store, due); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + due, + ) + .unwrap()); + assert_eq!(recovery(&store, &limited.id).attempt, 2); + assert_eq!(scheduler.network_turn(due), NetworkTurn::Resume); +} + +/// The failed start re-parks the row, then reschedules it. +fn fail_to_start(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + assert!(store + .transition_running_to_rate_limited( + session_id, + Some("Can't reach the API server"), + None, + Some(NETWORK_PAUSE_KIND), + ) + .unwrap()); + reschedule_after_start_failure(store, scheduler, session_id, now); +} + +/// A start failure says nothing about the network: the probe is abandoned, +/// so the delay does not grow and another row probes at once, and the +/// failure is counted against the session that could not start. +#[test] +fn a_probe_that_fails_to_start_is_counted_and_hands_the_probe_over() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + fail_to_start(&store, &scheduler, &sessions[0], now); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 1); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert!(state.next_retry_at.unwrap() > now); + + // The next row probes on the same tick time, with no delay added. + assert_eq!(tick(&scheduler, &store, now), [sessions[1].clone()]); + // And when that probe fails on the network, the delay is still the + // base one: the failed start doubled nothing. + fail_on_network(&store, &scheduler, &sessions[1], now); + let due = state.next_retry_at.unwrap().max(now + BASE_PROBE_DELAY_MS); + assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert!(!tick(&scheduler, &store, due).is_empty()); +} + +/// A lone network-paused session whose resume never starts is bounded by +/// the attempt cap, as it was before the gate existed. +#[test] +fn a_lone_session_that_never_starts_gives_up_at_the_attempt_cap() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let mut now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + for attempt in 1..=MAX_AUTO_RETRY_ATTEMPTS { + assert_eq!(tick(&scheduler, &store, now), sessions, "attempt {attempt}"); + fail_to_start(&store, &scheduler, &sessions[0], now); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, attempt); + assert_eq!(state.gave_up_at, None); + now = state.next_retry_at.unwrap(); + } + + assert_eq!(tick(&scheduler, &store, now), sessions); + fail_to_start(&store, &scheduler, &sessions[0], now); + let state = recovery(&store, &sessions[0]); + assert_eq!(state.gave_up_at, Some(now)); + assert_eq!(state.reason.as_deref(), Some("retry_cap_exceeded")); + assert!(tick(&scheduler, &store, now + HOUR_MS).is_empty()); +} + +/// What a real failed Claude turn records is not a sign of the network: the +/// error text, the error result and the visible message all follow the +/// claim, and none of them may open the gate or get the failure counted. +#[test] +fn a_failed_claude_probe_is_not_taken_for_a_success() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 3, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + claude_turn_fails_offline(&store, &sessions[0]); + // A tick that lands between those rows and the pause write resumes + // nobody into the outage. + assert!(tick(&scheduler, &store, now).is_empty()); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Hold); + + let failed_at = now + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + + // The gate closed as after any failed probe, and the delay doubles. + assert!(tick(&scheduler, &store, failed_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + let second_at = failed_at + BASE_PROBE_DELAY_MS; + assert_eq!(tick(&scheduler, &store, second_at), [sessions[1].clone()]); + claude_turn_fails_offline(&store, &sessions[1]); + fail_on_network(&store, &scheduler, &sessions[1], second_at); + assert_eq!(recovery(&store, &sessions[1]).attempt, 0); + assert!(tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, second_at + 2 * BASE_PROBE_DELAY_MS), + [sessions[2].clone()] + ); +} + +/// The signs that do open the gate: a result the agent did not mark as an +/// error, and a prompt response. Message text alone does not. +#[test] +fn a_successful_result_or_prompt_response_opens_the_gate_and_text_does_not() { + for kind in ["response_result", "prompt_response"] { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + store + .add_session_message(&sessions[0], MessageRole::Assistant, "Checking my work.") + .unwrap(); + acp_event( + &store, + &sessions[0], + "agent_message_chunk", + serde_json::json!({ "content": { "type": "text", "text": "Checking my work." } }), + ); + assert!(tick(&scheduler, &store, now).is_empty(), "{kind}"); + + let content = if kind == "response_result" { + serde_json::json!({ "type": "result", "subtype": "success", "is_error": false }) + } else { + serde_json::json!({ "stopReason": "end_turn" }) + }; + acp_event(&store, &sessions[0], kind, content); + assert_eq!( + tick(&scheduler, &store, now), + [sessions[1].clone()], + "{kind}" + ); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Resume); + } +} + +/// After an outage longer than 24 hours, a session's first counted failure +/// schedules a retry: the window starts there, not at the first pause. +#[test] +fn a_counted_failure_after_a_long_outage_does_not_give_up() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + // The ticks run in the past so that the probe's reply, stamped with + // the wall clock, lands after the probe's claim. + let paused_at = crate::store::now_timestamp() - 40 * HOUR_MS; + let sessions = network_paused_sessions(&store, &scheduler, 1, paused_at); + + let resumed_at = paused_at + 30 * HOUR_MS; + assert_eq!(tick(&scheduler, &store, resumed_at), sessions); + // The probe gets through, then the connection drops again. + agent_replies(&store, &sessions[0]); + let failed_at = resumed_at + 60_000; + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 1); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("backoff")); + assert_eq!(state.first_limited_at, failed_at); +} + +/// A probe that stays `running` with no sign of the provider stops holding +/// the other rows after the timeout. +#[test] +fn a_probe_that_hangs_times_out_and_another_row_probes() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS - 1).is_empty()); + assert_eq!( + scheduler.network_gate.lock().unwrap().probe(), + Some((sessions[0].clone(), now)) + ); + + // The timeout closes the gate like a failed probe: the next one waits + // the probe delay. + let timed_out_at = now + PROBE_TIMEOUT_MS; + assert!(tick(&scheduler, &store, timed_out_at).is_empty()); + assert_eq!(scheduler.network_gate.lock().unwrap().probe(), None); + assert!(tick(&scheduler, &store, timed_out_at + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, timed_out_at + BASE_PROBE_DELAY_MS), + [sessions[1].clone()] + ); + + // The session that hung is still running. When it fails at last, it is + // not counted and does not disturb the probe now running. + let late = timed_out_at + BASE_PROBE_DELAY_MS + 1_000; + fail_on_network(&store, &scheduler, &sessions[0], late); + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); + assert!(scheduler + .network_gate + .lock() + .unwrap() + .is_probing(&sessions[1])); +} + +/// The user's "Resume now" on a network-paused row: the claim the manual +/// resume makes, with the word it gives the scheduler. +fn resume_manually(store: &Store, scheduler: &RateLimitScheduler, session_id: &str, now: i64) { + let paused = store.get_session(session_id).unwrap().unwrap(); + assert!(store + .transition_to_running_clearing_agent_id(session_id, None) + .unwrap()); + scheduler.manual_resume_started(&paused, now); +} + +#[test] +fn a_manual_resume_that_gets_through_opens_the_gate() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 4, now); + // A failed probe leaves the gate closed with the next probe a delay away. + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + fail_on_network(&store, &scheduler, &sessions[0], now); + assert!(tick(&scheduler, &store, now).is_empty()); + + resume_manually(&store, &scheduler, &sessions[1], now); + // The gate watches that turn; nothing else starts meanwhile, even once + // the probe delay is up. + assert!(tick(&scheduler, &store, now + BASE_PROBE_DELAY_MS).is_empty()); + + agent_replies(&store, &sessions[1]); + assert_eq!(tick(&scheduler, &store, now + 1_000).len(), 2); + assert_eq!(scheduler.network_turn(now + 1_000), NetworkTurn::Resume); +} + +#[test] +fn a_manual_resume_that_fails_closes_the_gate_uncounted() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + fail_on_network(&store, &scheduler, &sessions[0], now); + + resume_manually(&store, &scheduler, &sessions[1], now); + claude_turn_fails_offline(&store, &sessions[1]); + fail_on_network(&store, &scheduler, &sessions[1], now); + + assert_eq!(recovery(&store, &sessions[1]).attempt, 0); + // It failed as the second probe in a row: the next waits twice the base. + assert!(tick(&scheduler, &store, now + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, now + 2 * BASE_PROBE_DELAY_MS).len(), + 1 + ); +} + +/// Only a network pause is watched, and only by a closed gate. +#[test] +fn a_manual_resume_of_another_row_tells_the_gate_nothing() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + + let limited = running_session(&store); + assert!(pause_for_rate_limit( + &store, + Some(&scheduler), + &limited.id, + Some("rate limit"), + None, + Some("claude_pro"), + now, + ) + .unwrap()); + resume_manually(&store, &scheduler, &limited.id, now); + assert_eq!(scheduler.network_turn(now), NetworkTurn::Probe); + + // A probe is already running: it keeps the gate. + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!(tick(&scheduler, &store, now), [sessions[0].clone()]); + resume_manually(&store, &scheduler, &sessions[1], now); + assert!(scheduler + .network_gate + .lock() + .unwrap() + .is_probing(&sessions[0])); +} + +#[test] +fn the_probe_delay_doubles_to_a_ten_minute_cap() { + let mut gate = NetworkGate::default(); + let mut now = 1_000; + let mut delays = Vec::new(); + for _ in 0..8 { + assert_eq!(gate.turn(now), NetworkTurn::Probe); + gate.probe_started("probe", now); + assert_eq!(gate.record_failure("probe", now), PauseAccounting::Waiting); + delays.push(gate.next_probe_at - now); + now = gate.next_probe_at; + } + assert_eq!( + delays, + [15_000, 30_000, 60_000, 120_000, 240_000, 480_000, 600_000, 600_000] + ); + + // An open period starts the doubling again. + gate.probe_started("probe", now); + gate.probe_succeeded("probe"); + assert_eq!(gate.record_failure("other", now), PauseAccounting::Counted); + assert_eq!(gate.next_probe_at - now, BASE_PROBE_DELAY_MS); + now = gate.next_probe_at; + gate.probe_started("probe", now); + assert_eq!(gate.record_failure("probe", now), PauseAccounting::Waiting); + assert_eq!(gate.next_probe_at - now, 2 * BASE_PROBE_DELAY_MS); +} + +#[test] +fn another_sessions_failure_leaves_a_running_probe_alone() { + let mut gate = NetworkGate::default(); + gate.probe_started("probe", 1_000); + + assert_eq!( + gate.record_failure("other", 2_000), + PauseAccounting::Waiting + ); + assert!(gate.is_probing("probe")); + + // Nor does a stale answer about a session that is not the probe move it. + gate.probe_succeeded("other"); + gate.probe_abandoned("other", 2_000); + assert!(gate.is_probing("probe")); +} + +/// A closed gate that has read a fingerprint with one address. +fn gate_with_a_network() -> NetworkGate { + let mut gate = NetworkGate::default(); + assert!(!gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()]))); + gate +} + +fn fail_a_probe(gate: &mut NetworkGate, session_id: &str, now: i64) -> PauseAccounting { + gate.probe_started(session_id, now); + gate.record_failure(session_id, now) +} + +#[test] +fn a_sessions_twentieth_failed_probe_in_a_row_exhausts_it() { + let mut gate = gate_with_a_network(); + for failure in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting, + "failure {failure}" + ); + // Another session's failed probes are its own. + assert_eq!( + fail_a_probe(&mut gate, "other", 1_000), + PauseAccounting::Waiting + ); + } + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::ProbesExhausted + ); + // The count starts over for a session that was given up on. + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting + ); +} + +#[test] +fn a_probe_failure_counts_only_awake_with_a_network_present() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + + // The machine slept while the probe ran. + gate.observe_tick(1_000); + gate.probe_started("lone", 1_000); + assert!(gate.observe_tick(1_000 + WAKE_GAP_MS + 1)); + assert_eq!( + gate.record_failure("lone", 2_000_000), + PauseAccounting::Waiting + ); + + // No interface holds an address. + assert!(gate.observe_fingerprint(BTreeSet::new())); + assert_eq!( + fail_a_probe(&mut gate, "lone", 3_000_000), + PauseAccounting::Waiting + ); + + // Nothing is known about the network. + gate.fingerprint_unreadable(); + assert_eq!( + fail_a_probe(&mut gate, "lone", 4_000_000), + PauseAccounting::Waiting + ); + + // None of those broke the run or added to it: the next failure awake + // with a network is the twentieth. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe(&mut gate, "lone", 5_000_000), + PauseAccounting::ProbesExhausted + ); +} + +#[test] +fn a_probe_that_gets_through_clears_the_failure_counts() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + gate.probe_started("other", 1_000); + gate.probe_succeeded("other"); + assert_eq!(gate.record_failure("lone", 1_000), PauseAccounting::Counted); + + // Closed again: the fingerprint is read anew, and "lone" starts at one. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe(&mut gate, "lone", 1_000), + PauseAccounting::Waiting + ); +} + +#[test] +fn a_timed_out_probe_closes_the_gate_like_a_failed_one_but_is_not_counted() { + let mut gate = gate_with_a_network(); + gate.probe_started("probe", 1_000); + gate.probe_timed_out("other", 2_000); + assert!(gate.is_probing("probe")); + + gate.probe_timed_out("probe", 2_000); + assert!(gate.is_closed()); + assert_eq!(gate.next_probe_at, 2_000 + BASE_PROBE_DELAY_MS); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + assert!(gate.probe_failures.is_empty()); +} + +/// The gate stopped waiting for `session_id`'s probe, and only then did the +/// session fail on the network, at `failed_at`. +fn fail_a_probe_late(gate: &mut NetworkGate, session_id: &str, failed_at: i64) -> PauseAccounting { + let started_at = failed_at - PROBE_TIMEOUT_MS - HOUR_MS; + gate.probe_started(session_id, started_at); + gate.probe_timed_out(session_id, started_at + PROBE_TIMEOUT_MS); + gate.record_failure(session_id, failed_at) +} + +/// A session whose probes always fail after the probe timeout is bounded +/// like one whose probes fail in time. +#[test] +fn a_timed_out_probe_that_fails_late_is_charged_to_the_sessions_probe_failures() { + let mut gate = gate_with_a_network(); + let mut now = 10 * HOUR_MS; + for failure in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!( + fail_a_probe_late(&mut gate, "lone", now), + PauseAccounting::Waiting, + "failure {failure}" + ); + assert_eq!(gate.probe_failures["lone"], failure); + now += 2 * HOUR_MS; + } + assert_eq!( + fail_a_probe_late(&mut gate, "lone", now), + PauseAccounting::ProbesExhausted + ); + assert!(gate.timed_out_probes.is_empty()); +} + +/// The late failure finds the gate closed with its probe time long past. +/// The next probe still waits the probe delay, which is not doubled again. +#[test] +fn a_late_probe_failure_does_not_start_the_next_probe_at_once() { + let mut gate = gate_with_a_network(); + gate.probe_started("lone", 1_000); + let timed_out_at = 1_000 + PROBE_TIMEOUT_MS; + gate.probe_timed_out("lone", timed_out_at); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + + let failed_at = timed_out_at + HOUR_MS; + assert_eq!(gate.turn(failed_at), NetworkTurn::Probe); + assert_eq!( + gate.record_failure("lone", failed_at), + PauseAccounting::Waiting + ); + + assert_eq!(gate.turn(failed_at), NetworkTurn::Hold); + assert_eq!(gate.next_probe_at, failed_at + 2 * BASE_PROBE_DELAY_MS); + assert_eq!(gate.probe_delay_ms, 2 * BASE_PROBE_DELAY_MS); + assert_eq!(gate.turn(gate.next_probe_at), NetworkTurn::Probe); +} + +/// A late failure never brings the next probe forward. +#[test] +fn a_late_probe_failure_keeps_a_later_probe_time() { + let mut gate = gate_with_a_network(); + gate.probe_started("lone", 1_000); + let timed_out_at = 1_000 + PROBE_TIMEOUT_MS; + gate.probe_timed_out("lone", timed_out_at); + gate.probe_delay_ms = 1_000; + + gate.record_failure("lone", timed_out_at + 1); + assert_eq!(gate.next_probe_at, timed_out_at + BASE_PROBE_DELAY_MS); +} + +#[test] +fn a_late_probe_failure_counts_only_awake_with_a_network_present() { + let mut gate = gate_with_a_network(); + for _ in 1..MAX_CONSECUTIVE_PROBE_FAILURES { + fail_a_probe(&mut gate, "lone", 1_000); + } + + // The machine slept after the gate stopped waiting for the probe. + gate.observe_tick(1_000); + gate.probe_started("lone", 1_000); + gate.probe_timed_out("lone", 1_000 + PROBE_TIMEOUT_MS); + gate.observe_tick(1_000 + PROBE_TIMEOUT_MS); + assert!(gate.observe_tick(1_000 + PROBE_TIMEOUT_MS + WAKE_GAP_MS + 1)); + assert_eq!( + gate.record_failure("lone", 2 * HOUR_MS), + PauseAccounting::Waiting + ); + + // It slept while the probe ran, before the timeout. + gate.observe_tick(3 * HOUR_MS); + gate.probe_started("lone", 3 * HOUR_MS); + assert!(gate.observe_tick(3 * HOUR_MS + PROBE_TIMEOUT_MS)); + gate.probe_timed_out("lone", 3 * HOUR_MS + PROBE_TIMEOUT_MS); + assert_eq!( + gate.record_failure("lone", 4 * HOUR_MS), + PauseAccounting::Waiting + ); + + // No interface holds an address. + assert!(gate.observe_fingerprint(BTreeSet::new())); + assert_eq!( + fail_a_probe_late(&mut gate, "lone", 6 * HOUR_MS), + PauseAccounting::Waiting + ); + + // None of those added to the run: the next late failure awake with a + // network is the twentieth. + gate.observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!( + fail_a_probe_late(&mut gate, "lone", 8 * HOUR_MS), + PauseAccounting::ProbesExhausted + ); +} + +/// While another session probes, the late failure is charged and the +/// running probe is left alone. +#[test] +fn a_late_probe_failure_during_another_probe_is_charged_and_leaves_it_alone() { + let mut gate = gate_with_a_network(); + gate.probe_started("hung", 1_000); + gate.probe_timed_out("hung", 1_000 + PROBE_TIMEOUT_MS); + let probing_at = gate.next_probe_at; + gate.probe_started("other", probing_at); + + assert_eq!( + gate.record_failure("hung", probing_at + 1_000), + PauseAccounting::Waiting + ); + assert_eq!(gate.probe_failures["hung"], 1); + assert!(gate.is_probing("other")); +} + +/// Once a probe gets through, an earlier timed-out probe is forgotten: its +/// failure with the gate open is counted like any other. +#[test] +fn a_probe_that_gets_through_forgets_the_timed_out_probes() { + let mut gate = gate_with_a_network(); + gate.probe_started("hung", 1_000); + gate.probe_timed_out("hung", 1_000 + PROBE_TIMEOUT_MS); + gate.probe_started("other", gate.next_probe_at); + gate.probe_succeeded("other"); + assert!(gate.timed_out_probes.is_empty()); + + assert_eq!( + gate.record_failure("hung", 2 * HOUR_MS), + PauseAccounting::Counted + ); + assert!(gate.probe_failures.is_empty()); +} + +/// A lone session whose probe fails after the timeout, through the +/// scheduler: the failure is charged, and the row, due at once, is not +/// probed again until the probe delay has passed. +#[test] +fn a_lone_session_that_fails_after_the_probe_timeout_waits_the_probe_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = RateLimitScheduler::new(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + scheduler + .network_gate + .lock() + .unwrap() + .observe_fingerprint(BTreeSet::from(["192.0.2.10".parse().unwrap()])); + assert_eq!(tick(&scheduler, &store, now), sessions); + assert!(tick(&scheduler, &store, now + PROBE_TIMEOUT_MS).is_empty()); + assert_eq!(scheduler.network_gate.lock().unwrap().probe(), None); + + let failed_at = now + PROBE_TIMEOUT_MS + HOUR_MS; + claude_turn_fails_offline(&store, &sessions[0]); + fail_on_network(&store, &scheduler, &sessions[0], failed_at); + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + assert_eq!(state.next_retry_at, Some(failed_at)); + assert_eq!( + scheduler.network_gate.lock().unwrap().probe_failures[&sessions[0]], + 1 + ); + assert!(tick(&scheduler, &store, failed_at).is_empty()); + assert!(tick(&scheduler, &store, failed_at + 2 * BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick(&scheduler, &store, failed_at + 2 * BASE_PROBE_DELAY_MS), + sessions + ); +} diff --git a/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs new file mode 100644 index 000000000..8a8087efb --- /dev/null +++ b/apps/staged/src-tauri/src/rate_limit_scheduler/network_gate/tests/triggers.rs @@ -0,0 +1,421 @@ +//! The wake and network-change triggers, driven tick by tick with an +//! injected clock and a fake fingerprint. + +use std::collections::BTreeSet; +use std::io; +use std::net::IpAddr; +use std::sync::Mutex; + +use super::*; + +/// The scheduler's tick interval. +const TICK_MS: i64 = 15_000; +/// A gap between ticks that only a suspended process produces. +const SUSPENDED_MS: i64 = WAKE_GAP_MS + 1; + +#[derive(Default)] +struct FakeNetworkState { + /// `None` makes the read fail. + addresses: Option>, + reads: usize, +} + +/// A fingerprint the test sets by hand, which counts how often it is read. +#[derive(Clone)] +struct FakeNetwork(Arc>); + +impl FakeNetwork { + fn with_addresses(addresses: &[&str]) -> Self { + let network = Self(Arc::default()); + network.set_addresses(addresses); + network + } + + fn set_addresses(&self, addresses: &[&str]) { + self.0.lock().unwrap().addresses = Some( + addresses + .iter() + .map(|address| address.parse().unwrap()) + .collect(), + ); + } + + fn make_unreadable(&self) { + self.0.lock().unwrap().addresses = None; + } + + fn reads(&self) -> usize { + self.0.lock().unwrap().reads + } + + fn scheduler(&self) -> RateLimitScheduler { + RateLimitScheduler::with_network_fingerprint(Box::new(self.clone())) + } +} + +impl NetworkFingerprint for FakeNetwork { + fn read(&self) -> io::Result> { + let mut state = self.0.lock().unwrap(); + state.reads += 1; + state + .addresses + .clone() + .ok_or_else(|| io::Error::other("interfaces unavailable")) + } +} + +const WIFI: &[&str] = &["192.0.2.10"]; +const OTHER_WIFI: &[&str] = &["198.51.100.7"]; + +/// Tick every 15 seconds from `from`, failing each probe as it starts, until +/// four have failed. The fourth fails 105 seconds in and leaves the next +/// probe two minutes away. Returns the time of the last tick. +fn fail_four_probes(scheduler: &RateLimitScheduler, store: &Arc, from: i64) -> i64 { + let mut at = from; + let mut failures = 0; + loop { + for session_id in tick_with_triggers(scheduler, store, at) { + fail_on_network(store, scheduler, &session_id, at); + failures += 1; + } + if failures == 4 { + assert_eq!(at, from + 7 * TICK_MS); + return at; + } + at += TICK_MS; + } +} + +#[test] +fn a_long_gap_resets_the_probe_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + // The next probe is two minutes off, so an ordinary tick starts nothing. + let before_sleep = last + TICK_MS; + assert!(tick_with_triggers(&scheduler, &store, before_sleep).is_empty()); + + // The machine sleeps. The tick that sees the gap starts nothing, and the + // one after it probes, long before the two minutes are up. + let woke = before_sleep + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); + let probed = woke + TICK_MS; + let probe = tick_with_triggers(&scheduler, &store, probed); + assert_eq!(probe.len(), 1); + + // The doubling starts over too: a failure waits the base delay. + fail_on_network(&store, &scheduler, &probe[0], probed); + assert!(tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS).len(), + 1 + ); +} + +#[test] +fn no_probe_starts_on_the_tick_after_a_long_gap() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + // A probe is due at once: the gate is fresh. + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let woke = now + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, woke + TICK_MS), + sessions + ); +} + +#[test] +fn a_gap_of_exactly_the_threshold_is_normal() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + assert_eq!( + tick_with_triggers(&scheduler, &store, now + WAKE_GAP_MS), + sessions + ); +} + +#[test] +fn a_run_of_long_gaps_never_starts_a_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + // Maintenance wakes during a night of sleep: one tick each, far apart. + let mut at = now; + for _ in 0..20 { + at += 20 * 60_000; + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + } + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); + + assert_eq!( + tick_with_triggers(&scheduler, &store, at + TICK_MS), + sessions + ); +} + +#[test] +fn a_fingerprint_change_resets_the_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + // The same network: the two minute wait stands. + assert!(tick_with_triggers(&scheduler, &store, last + TICK_MS).is_empty()); + + network.set_addresses(OTHER_WIFI); + let probed = last + 2 * TICK_MS; + let probe = tick_with_triggers(&scheduler, &store, probed); + assert_eq!(probe.len(), 1); + + fail_on_network(&store, &scheduler, &probe[0], probed); + assert!(tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS - 1).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, probed + BASE_PROBE_DELAY_MS).len(), + 1 + ); +} + +/// Tick every 15 seconds from `from` until a probe starts, and return it +/// with the time it started. +fn next_probe(scheduler: &RateLimitScheduler, store: &Arc, from: i64) -> (String, i64) { + let mut at = from; + loop { + let mut started = tick_with_triggers(scheduler, store, at); + if let Some(session_id) = started.pop() { + assert!(started.is_empty()); + return (session_id, at); + } + at += TICK_MS; + assert!(at < from + 2 * MAX_PROBE_DELAY_MS, "no probe started"); + } +} + +/// No address at all is not a reason to stop probing: the provider may be +/// on this machine. The doubling schedule runs as it does with a network. +#[test] +fn an_empty_fingerprint_does_not_stop_the_doubling_probe_schedule() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + let mut waits = Vec::new(); + for _ in 0..4 { + let (probe, started_at) = next_probe(&scheduler, &store, at); + assert_eq!(probe, sessions[0]); + waits.push(started_at - at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; + } + assert_eq!(waits, [0, 15_000, 30_000, 60_000]); + assert_eq!(recovery(&store, &sessions[0]).attempt, 0); +} + +/// Losing every address is a change to an empty fingerprint, which resets +/// nothing, and a probe that comes due meanwhile still starts. +#[test] +fn losing_every_address_neither_resets_the_delay_nor_blocks_a_due_probe() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + // The next probe is two minutes off. The addresses go: no early probe. + network.set_addresses(&[]); + let due = last + 120_000; + let mut at = last + TICK_MS; + while at < due { + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + at += TICK_MS; + } + // Due, with still no address: it starts all the same. + assert_eq!(tick_with_triggers(&scheduler, &store, due).len(), 1); +} + +/// An address appearing after there was none is a change that does reset. +#[test] +fn an_address_appearing_resets_the_delay() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(&[]); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + assert!(tick_with_triggers(&scheduler, &store, last + TICK_MS).is_empty()); + + network.set_addresses(WIFI); + assert_eq!( + tick_with_triggers(&scheduler, &store, last + 2 * TICK_MS).len(), + 1 + ); +} + +/// A wake with no network has nothing new to probe: the schedule stands. +#[test] +fn a_wake_with_no_network_leaves_the_probe_schedule_alone() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + let due = last + 120_000; + + let woke = last + TICK_MS + SUSPENDED_MS; + let mut at = woke; + while at < due { + assert!( + tick_with_triggers(&scheduler, &store, at).is_empty(), + "{} ms before the probe is due", + due - at + ); + at += TICK_MS; + } + assert_eq!(tick_with_triggers(&scheduler, &store, at).len(), 1); +} + +/// A read that fails says nothing about the network, so it does not hold a +/// wake back the way an empty fingerprint does. +#[test] +fn an_unreadable_fingerprint_does_not_hold_back_a_wake() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(&[]); + let scheduler = network.scheduler(); + let now = crate::store::now_timestamp(); + network_paused_sessions(&store, &scheduler, 2, now); + let last = fail_four_probes(&scheduler, &store, now); + + network.make_unreadable(); + let woke = last + TICK_MS + SUSPENDED_MS; + assert!(tick_with_triggers(&scheduler, &store, woke).is_empty()); + assert_eq!( + tick_with_triggers(&scheduler, &store, woke + TICK_MS).len(), + 1 + ); +} + +/// The whole path for a lone session behind a dead proxy: the machine is +/// awake with an address, every probe fails, and after the twentieth the +/// session is left paused for a manual resume. +#[test] +fn a_lone_session_gives_up_after_twenty_failed_probes_with_a_network() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(WIFI).scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + for failure in 1..=MAX_CONSECUTIVE_PROBE_FAILURES { + let (probe, started_at) = next_probe(&scheduler, &store, at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; + + let state = recovery(&store, &sessions[0]); + assert_eq!(state.attempt, 0); + if failure < MAX_CONSECUTIVE_PROBE_FAILURES { + assert_eq!(state.gave_up_at, None, "failure {failure}"); + assert_eq!(state.reason.as_deref(), Some("network_wait")); + } else { + assert_eq!(state.gave_up_at, Some(at)); + assert_eq!(state.next_retry_at, None); + assert_eq!(state.reason.as_deref(), Some("probe_limit_exceeded")); + } + } + + // Still paused, and never probed with again. + let row = store.get_session(&sessions[0]).unwrap().unwrap(); + assert_eq!(row.status, SessionStatus::Paused); + let end = at + 2 * MAX_PROBE_DELAY_MS; + while at < end { + at += TICK_MS; + assert!(tick_with_triggers(&scheduler, &store, at).is_empty()); + } +} + +/// With no network the same run of failures gives up on nothing. +#[test] +fn failed_probes_with_no_network_never_give_up() { + let store = Arc::new(Store::in_memory().unwrap()); + let scheduler = FakeNetwork::with_addresses(&[]).scheduler(); + let now = crate::store::now_timestamp(); + let sessions = network_paused_sessions(&store, &scheduler, 1, now); + + let mut at = now; + for _ in 0..MAX_CONSECUTIVE_PROBE_FAILURES + 5 { + let (probe, started_at) = next_probe(&scheduler, &store, at); + fail_on_network(&store, &scheduler, &probe, started_at); + at = started_at; + } + let state = recovery(&store, &sessions[0]); + assert_eq!(state.gave_up_at, None); + assert_eq!(state.reason.as_deref(), Some("network_wait")); +} + +#[test] +fn the_fingerprint_is_read_only_while_closed_with_network_rows() { + let store = Arc::new(Store::in_memory().unwrap()); + let network = FakeNetwork::with_addresses(WIFI); + let scheduler = network.scheduler(); + // The store stamps the probe's reply with the wall clock, so the ticks + // run in the past: the reply then lands after the probe's claim. + let now = crate::store::now_timestamp() - 10 * TICK_MS; + + // Closed, but nothing is paused on the network. + assert!(tick_with_triggers(&scheduler, &store, now).is_empty()); + assert_eq!(network.reads(), 0); + + // Closed with a row waiting: read, and the row probes. + let sessions = network_paused_sessions(&store, &scheduler, 2, now); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + TICK_MS), + [sessions[0].clone()] + ); + assert_eq!(network.reads(), 1); + + // Probing: not read. + assert!(tick_with_triggers(&scheduler, &store, now + 2 * TICK_MS).is_empty()); + assert_eq!(network.reads(), 1); + + // Open: not read. + agent_replies(&store, &sessions[0]); + assert_eq!( + tick_with_triggers(&scheduler, &store, now + 3 * TICK_MS), + [sessions[1].clone()] + ); + assert_eq!(network.reads(), 1); +} + +#[test] +fn the_real_fingerprint_leaves_out_loopback_and_link_local_addresses() { + let Ok(addresses) = InterfaceAddresses.read() else { + return; + }; + for address in addresses { + assert!(!address.is_loopback(), "{address}"); + let link_local = match address { + IpAddr::V4(v4) => v4.is_link_local(), + IpAddr::V6(v6) => (v6.segments()[0] & 0xffc0) == 0xfe80, + }; + assert!(!link_local, "{address}"); + } +} diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index cd598781b..1affb0c26 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -1788,6 +1788,20 @@ pub(crate) struct RateLimitResumeTicket { pub rate_limit_kind: Option, } +impl RateLimitResumeTicket { + /// What the resume sends: the user's queued message when there is one, + /// otherwise the synthetic prompt for what paused the session. + pub(crate) fn prompt(&self) -> String { + match &self.queued_message { + Some(message) => message.content.clone(), + None => { + crate::rate_limit_scheduler::auto_resume_prompt(self.rate_limit_kind.as_deref()) + .to_string() + } + } + } +} + pub(crate) async fn auto_resume_rate_limited_session( store: Arc, registry: Arc, @@ -1797,19 +1811,14 @@ pub(crate) async fn auto_resume_rate_limited_session( ticket: RateLimitResumeTicket, ) -> Result<(), String> { // The post-success drain cascade delivers any remaining queued messages. - let (prompt, image_ids, branch_id, queued_message_id) = match &ticket.queued_message { + let prompt = ticket.prompt(); + let (image_ids, branch_id, queued_message_id) = match &ticket.queued_message { Some(message) => ( - message.content.clone(), Some(message.image_ids.clone()), message.branch_id.clone(), Some(message.id.clone()), ), - None => ( - crate::rate_limit_scheduler::AUTO_RETRY_RESUME_PROMPT.to_string(), - None, - None, - None, - ), + None => (None, None, None), }; let result = resume_session_for_store( @@ -1905,6 +1914,13 @@ pub(crate) async fn resume_session_for_store( if !transitioned { return Err(SESSION_ALREADY_RUNNING_ERROR.to_string()); } + // A manual resume of a network-paused session doubles as the + // network gate's probe, so getting through resumes the rest. + if let Some(scheduler) = + app_handle.try_state::>() + { + scheduler.manual_resume_started(&session, store::now_timestamp()); + } } // The caller's claim predates `prepare_resume`, and the hold that // keeps the row live is invisible to a Stop: one that landed during diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index e9260c765..4bdf42da0 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -38,7 +38,9 @@ pub(crate) mod attachments; mod disconnect; +mod failed_run; mod late_stop; +mod network_failure; pub(crate) mod reconcile; pub(crate) use disconnect::may_still_run_on_workstation; @@ -1792,6 +1794,7 @@ pub fn start_session( // connection, so a remote turn may still be running behind it. Reset // with `settle_reason`. let cancel_unacknowledged = Cell::new(false); + let prompt_failed = Cell::new(false); // Whether the last connection settled out of its background hold with // the work still going (see `SessionSettled::background_work_left`). // Reset with `settle_reason`. @@ -2383,6 +2386,7 @@ pub fn start_session( settle_reason.set(None); transport_loss.set(None); cancel_unacknowledged.set(false); + prompt_failed.set(false); background_work_left.set(false); reattached.set(false); let turn_result = match driver @@ -2450,6 +2454,7 @@ pub fn start_session( settle_reason.set(Some(settled.reason)); transport_loss.set(settled.transport_loss); cancel_unacknowledged.set(settled.cancel_unacknowledged); + prompt_failed.set(settled.prompt_failed); background_work_left.set(settled.background_work_left); match settled.reason { acp_client::SessionSettleReason::HeldUntilCap => { @@ -2545,6 +2550,7 @@ pub fn start_session( settle_reason.set(settlement.settle_reason); transport_loss.set(settlement.transport_loss); cancel_unacknowledged.set(settlement.cancel_unacknowledged); + prompt_failed.set(settlement.prompt_failed); background_work_left.set(settlement.background_work_left); reattached.set(true); // The agent went on with the turn after anything read @@ -2756,6 +2762,7 @@ pub fn start_session( transport_loss: transport_loss.get(), cancel_unacknowledged: cancel_unacknowledged.get(), background_work_left: background_work_left.get(), + prompt_failed: prompt_failed.get(), }; let reattached = reattached.get(); @@ -3018,6 +3025,10 @@ struct RunSettlement { /// The connection settled out of its background hold with the work still /// going (see `SessionSettled::background_work_left`). background_work_left: bool, + /// The agent answered the run's last prompt with an error of its own + /// (see `SessionSettled::prompt_failed`). A turn reattached to was not + /// prompted by the connection that saw it end, so it has the default. + prompt_failed: bool, } /// The transport verdict of a turn reattached to after its connection went: @@ -3307,45 +3318,45 @@ fn finish_session_run( .get_session(&session_id_for_status) .ok() .flatten(); - let provider_metadata = session_snapshot - .as_ref() - .and_then(|session| session.provider_metadata.as_ref()); let provider = session_snapshot .as_ref() .and_then(|session| session.provider.as_deref()) .or(config.provider.as_deref()); - // A rate limit pauses the session; anything else fails it, as a - // lost remote agent when that is what the error says. - let completion_reason = - match completion_reason_for_driver_error(message, provider_metadata) { - CompletionReason::Crashed => failed_run_completion_reason(message), - reason => reason, - }; - let rate_limit_recovery = - (completion_reason == CompletionReason::RateLimited).then(|| { - rate_limit_recovery_details( - message, - provider_metadata, - crate::store::now_timestamp(), - ) - }); - if let Some(recovery) = rate_limit_recovery.as_ref() { - crate::rate_limit_metrics::record_rate_limit_hit( + // A provider limit or a network failure pauses the session; + // anything else fails it (see `failed_run`). + let decision = failed_run::decide_failed_run( + session_snapshot.as_ref(), + config.provider.as_deref(), + message, + settlement.prompt_failed, + crate::store::now_timestamp(), + ); + match (decision.network_failure, decision.recovery.as_ref()) { + (Some(failure), _) => { + log::warn!( + "Session {session_id_for_status}: the prompt failed on the network \ + ({}); pausing it for an automatic resume", + failure.signal() + ); + crate::rate_limit_metrics::record_network_pause( + &session_id_for_status, + provider, + failure.signal(), + ); + } + (None, Some(recovery)) => crate::rate_limit_metrics::record_rate_limit_hit( &session_id_for_status, provider, recovery.kind.as_deref(), recovery.resets_at.is_some(), - ); + ), + (None, None) => {} } ( - if completion_reason == CompletionReason::RateLimited { - "paused" - } else { - "error" - }, + decision.status(), Some(innermost_error_sentence(message).unwrap_or_else(|| message.clone())), - completion_reason, - rate_limit_recovery, + decision.completion_reason, + decision.recovery, ) } }; @@ -3484,8 +3495,14 @@ fn finish_session_run( // Status flip and retry bookkeeping land in one write, so a // scheduler tick can never see a paused row without bookkeeping // and double-count the attempt. + use tauri::Manager; + let rate_limit_scheduler = + app_handle.try_state::>(); crate::rate_limit_scheduler::pause_for_rate_limit( &store_for_status, + rate_limit_scheduler + .as_deref() + .map(|scheduler| &**scheduler), &session_id_for_status, error_msg.as_deref(), recovery.resets_at.as_deref(), @@ -11800,6 +11817,7 @@ draft file" transport_loss: None, cancel_unacknowledged, background_work_left: false, + prompt_failed: false, } } diff --git a/apps/staged/src-tauri/src/session_runner/failed_run.rs b/apps/staged/src-tauri/src/session_runner/failed_run.rs new file mode 100644 index 000000000..c3efb3b93 --- /dev/null +++ b/apps/staged/src-tauri/src/session_runner/failed_run.rs @@ -0,0 +1,330 @@ +//! How a run that ended in an agent error is recorded. +//! +//! Three outcomes, tried in this order: +//! +//! 1. **A provider limit** pauses the session until the limit resets. +//! 2. **A network failure** pauses it too, under the same pause with the +//! kind [`NETWORK_PAUSE_KIND`], so the scheduler resumes it. Pipeline +//! sessions are left out: a push or pull that fails on the network fails. +//! 3. **Anything else** fails the session, as a lost remote agent when that +//! is what the error says. +//! +//! The limit check runs first so a limit is never taken for the network. + +use super::network_failure::{classify_network_failure, NetworkFailure}; +use super::{ + completion_reason_for_driver_error, failed_run_completion_reason, rate_limit_recovery_details, + RateLimitRecovery, +}; +use crate::rate_limit_scheduler::NETWORK_PAUSE_KIND; +use crate::store::{CompletionReason, Session}; + +/// What [`decide_failed_run`] concluded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FailedRunDecision { + pub(super) completion_reason: CompletionReason, + /// Set exactly when the session is to be paused for an automatic resume. + pub(super) recovery: Option, + /// The signal that marked the failure as the network, when it paused the + /// session for that reason. + pub(super) network_failure: Option, +} + +impl FailedRunDecision { + /// The session status this decision is written as. + pub(super) fn status(&self) -> &'static str { + if self.recovery.is_some() { + "paused" + } else { + "error" + } + } +} + +/// Decide how a run that failed with `driver_error` ends. +/// +/// `session` is the row as it stands after the run, when it could be read; +/// `fallback_provider` is the run's configured provider, for a row that +/// records none; `prompt_failed` is the driver's flag that the agent answered +/// the prompt with an error (`acp_client::SessionSettled::prompt_failed`). +pub(super) fn decide_failed_run( + session: Option<&Session>, + fallback_provider: Option<&str>, + driver_error: &str, + prompt_failed: bool, + now_millis: i64, +) -> FailedRunDecision { + let provider_metadata = session.and_then(|session| session.provider_metadata.as_ref()); + if completion_reason_for_driver_error(driver_error, provider_metadata) + == CompletionReason::RateLimited + { + return FailedRunDecision { + completion_reason: CompletionReason::RateLimited, + recovery: Some(rate_limit_recovery_details( + driver_error, + provider_metadata, + now_millis, + )), + network_failure: None, + }; + } + + let provider = session + .and_then(|session| session.provider.as_deref()) + .or(fallback_provider); + let is_pipeline = session.is_some_and(|session| session.pipeline.is_some()); + let network_failure = if is_pipeline { + None + } else { + classify_network_failure(provider, prompt_failed, provider_metadata) + }; + if network_failure.is_some() { + return FailedRunDecision { + // The pause shared with provider limits; the kind tells them + // apart. No reset time: nothing says when the network returns. + completion_reason: CompletionReason::RateLimited, + recovery: Some(RateLimitRecovery { + resets_at: None, + kind: Some(NETWORK_PAUSE_KIND.to_string()), + }), + network_failure, + }; + } + + FailedRunDecision { + completion_reason: failed_run_completion_reason(driver_error), + recovery: None, + network_failure: None, + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serde_json::json; + + use super::*; + use crate::store::{PipelineExecution, PipelineKind, SessionStatus, Store}; + + const NOW_MS: i64 = 1_794_470_400_000; + const DNS_MESSAGE: &str = "Internal error: API Error: Can't reach the API server — check your internet or DNS (ENOTFOUND)"; + + /// The driver error as the runner receives it: the prompt failure wrapped + /// once by the connection that carried it. + fn driver_error(message: &str) -> String { + let inner = format!( + r#"Prompt failed: Error {{ code: -32603: Internal error, message: {message:?}, data: None }}"# + ); + format!( + r#"ACP protocol failed: Error {{ code: -32603: Internal error, message: "Internal error", data: Some(String({inner:?})) }}"# + ) + } + + fn prompt_error_metadata(kind: &str, message: &str) -> serde_json::Value { + json!({ "acp": { "promptError": { + "code": -32603, + "message": message, + "data": { "errorKind": kind }, + } } }) + } + + /// A Claude session whose prompt failed on a DNS lookup. + fn session_with_dns_failure() -> Session { + let mut session = Session::new_running("work", Path::new("/tmp")).with_provider("claude"); + session.provider_metadata = Some(prompt_error_metadata("server_error", DNS_MESSAGE)); + session + } + + #[test] + fn a_network_prompt_error_pauses_with_the_network_kind_and_no_reset() { + let session = session_with_dns_failure(); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); + + assert_eq!(decision.status(), "paused"); + assert_eq!(decision.completion_reason, CompletionReason::RateLimited); + assert_eq!( + decision.recovery, + Some(RateLimitRecovery { + resets_at: None, + kind: Some("network".to_string()), + }) + ); + assert_eq!( + decision.network_failure, + Some(NetworkFailure::ClaudeErrno("ENOTFOUND")) + ); + } + + /// The decision carried through the write `finish_session_run` makes with + /// it: the row is parked for the scheduler with kind `network`. + #[test] + fn a_network_prompt_error_parks_the_row_with_kind_network() { + let store = Store::in_memory().unwrap(); + let session = session_with_dns_failure().with_agent("agent-1"); + store.create_session(&session).unwrap(); + let row = store.get_session(&session.id).unwrap().unwrap(); + + let decision = + decide_failed_run(Some(&row), None, &driver_error(DNS_MESSAGE), true, NOW_MS); + let recovery = decision.recovery.expect("a network failure pauses"); + assert!(crate::rate_limit_scheduler::pause_for_rate_limit( + &store, + None, + &session.id, + Some("Can't reach the API server"), + recovery.resets_at.as_deref(), + recovery.kind.as_deref(), + NOW_MS, + ) + .unwrap()); + + let parked = store.get_session(&session.id).unwrap().unwrap(); + assert_eq!(parked.status, SessionStatus::Paused); + assert_eq!( + parked.completion_reason, + Some(CompletionReason::RateLimited) + ); + assert_eq!(parked.rate_limit_kind.as_deref(), Some("network")); + assert_eq!(parked.rate_limit_resets_at, None); + let listed = store.get_rate_limited_paused_sessions().unwrap(); + assert_eq!(listed.len(), 1, "the scheduler lists the row"); + let bookkeeping = crate::rate_limit_scheduler::auto_recovery_from_metadata( + parked.provider_metadata.as_ref(), + ) + .expect("the pause writes retry bookkeeping"); + // With the network gate not open, the pause spends no attempt and + // waits for the gate. + assert_eq!(bookkeeping.attempt, 0); + assert_eq!(bookkeeping.next_retry_at, Some(NOW_MS)); + assert_eq!(bookkeeping.reason.as_deref(), Some("network_wait")); + assert!(!bookkeeping.unsafe_to_auto_resume); + } + + #[test] + fn a_pipeline_session_with_the_same_error_fails_as_crashed() { + for kind in [PipelineKind::Push, PipelineKind::Pull] { + let mut session = session_with_dns_failure(); + session.pipeline = Some(PipelineExecution::from_steps(&[]).with_kind(kind)); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); + + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + assert_eq!(decision.recovery, None); + assert_eq!(decision.network_failure, None); + } + } + + #[test] + fn the_configured_provider_stands_in_for_a_row_that_records_none() { + let mut session = session_with_dns_failure(); + session.provider = None; + let error = driver_error(DNS_MESSAGE); + + let with_fallback = decide_failed_run(Some(&session), Some("claude"), &error, true, NOW_MS); + assert_eq!(with_fallback.status(), "paused"); + + // Codex is never classified, and neither is an unknown agent. + for provider in [Some("codex"), None] { + let decision = decide_failed_run(Some(&session), provider, &error, true, NOW_MS); + assert_eq!(decision.status(), "error", "{provider:?}"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + } + } + + /// The limit check runs first and keeps its own kind and reset handling. + #[test] + fn a_rate_limit_is_still_a_rate_limit() { + let mut session = session_with_dns_failure(); + session.provider_metadata = Some(prompt_error_metadata( + "rate_limit", + "Internal error: rate limit exceeded", + )); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error("Internal error: rate limit exceeded"), + true, + NOW_MS, + ); + + assert_eq!(decision.status(), "paused"); + assert_eq!(decision.completion_reason, CompletionReason::RateLimited); + assert_eq!( + decision.recovery.and_then(|recovery| recovery.kind), + Some("rate_limit".to_string()) + ); + assert_eq!(decision.network_failure, None); + } + + /// The same row and error text, but the driver did not report a failed + /// prompt: the stored `promptError` is stale and the run fails. + #[test] + fn a_run_that_did_not_fail_on_the_prompt_is_not_paused() { + let session = session_with_dns_failure(); + + let decision = decide_failed_run( + Some(&session), + None, + &driver_error(DNS_MESSAGE), + false, + NOW_MS, + ); + + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + assert_eq!(decision.network_failure, None); + } + + #[test] + fn other_failures_still_fail_the_session() { + // A real 5xx: the same structured kind, no network token. + let mut session = session_with_dns_failure(); + let message = "Internal error: API Error: 500 Internal server error"; + session.provider_metadata = Some(prompt_error_metadata("server_error", message)); + let decision = + decide_failed_run(Some(&session), None, &driver_error(message), true, NOW_MS); + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + + // A row that could not be read has nothing to classify. + let decision = decide_failed_run( + None, + Some("claude"), + &driver_error(DNS_MESSAGE), + true, + NOW_MS, + ); + assert_eq!(decision.status(), "error"); + assert_eq!(decision.completion_reason, CompletionReason::Crashed); + + // A lost remote agent keeps its own reason. + let decision = decide_failed_run( + None, + Some("claude"), + "ACP protocol failed: server shut down unexpectedly", + false, + NOW_MS, + ); + assert_eq!(decision.status(), "error"); + assert_eq!( + decision.completion_reason, + failed_run_completion_reason("ACP protocol failed: server shut down unexpectedly") + ); + } +} diff --git a/apps/staged/src-tauri/src/session_runner/late_stop.rs b/apps/staged/src-tauri/src/session_runner/late_stop.rs index b98550e28..f9a9b53f5 100644 --- a/apps/staged/src-tauri/src/session_runner/late_stop.rs +++ b/apps/staged/src-tauri/src/session_runner/late_stop.rs @@ -9,9 +9,10 @@ //! - **Before the write** ([`terminal_decision_after_late_stop`]): the run //! reads the token once more and the Stop replaces whatever it had decided. //! - **After the write** ([`deregister_paused_run`]): for a terminal row there -//! is nothing left to stop. A row paused for rate-limit recovery is not -//! terminal — the scheduler restarts it — so the run reads the Stop out as -//! it deregisters and ends the row. +//! is nothing left to stop. A row paused for rate-limit recovery, or for +//! a network failure under the same pause, is not terminal — the scheduler +//! restarts it — so the run reads the Stop out as it deregisters and ends +//! the row. use super::{RateLimitRecovery, RegistrationEpoch, SessionRegistry}; use crate::store::{CompletionReason, SessionStatus, Store}; @@ -187,6 +188,16 @@ mod tests { } fn paused_run() -> PausedRun { + paused_run_for(Some(RESETS_AT), "five_hour") + } + + /// A run paused because its prompt failed on the network: the same pause + /// with the network kind and no reset time. + fn network_paused_run() -> PausedRun { + paused_run_for(None, crate::rate_limit_scheduler::NETWORK_PAUSE_KIND) + } + + fn paused_run_for(resets_at: Option<&str>, kind: &str) -> PausedRun { let store = Arc::new(Store::in_memory().unwrap()); let project = Project::new("test-owner/test-repo"); store.create_project(&project).unwrap(); @@ -207,10 +218,11 @@ mod tests { registry.mark_finished(&session.id, epoch); assert!(crate::rate_limit_scheduler::pause_for_rate_limit( &store, + None, &session.id, Some("rate limit"), - Some(RESETS_AT), - Some("five_hour"), + resets_at, + Some(kind), NOW_MS, ) .unwrap()); @@ -339,6 +351,62 @@ mod tests { } } + /// A network pause is the same pause, so a Stop on either side of its + /// write ends the row `cancelled` and the scheduler never restarts it. + #[test] + fn a_stop_during_a_network_pause_write_still_ends_cancelled() { + // Before the write: the Stop replaces the decision, recovery included. + let decided: TerminalDecision = ( + "paused", + Some("Can't reach the API server".to_string()), + CompletionReason::RateLimited, + Some(RateLimitRecovery { + resets_at: None, + kind: Some(crate::rate_limit_scheduler::NETWORK_PAUSE_KIND.to_string()), + }), + ); + assert_eq!( + terminal_decision_after_late_stop(decided, Some(CompletionReason::Interrupted)), + ("cancelled", None, CompletionReason::Interrupted, None) + ); + + // After the write: the deregister reads the Stop out and ends the row. + let run = network_paused_run(); + assert_eq!(run.row().status, SessionStatus::Paused); + assert_eq!(run.row().rate_limit_kind.as_deref(), Some("network")); + assert!(run.registry.cancel(&run.session_id)); + + assert!(run.deregister(&mock_app())); + + let row = run.row(); + assert_eq!(row.status, SessionStatus::Cancelled); + assert_eq!(row.completion_reason, Some(CompletionReason::Interrupted)); + assert!( + run.store + .get_rate_limited_paused_sessions() + .unwrap() + .is_empty(), + "the scheduler no longer lists the row" + ); + } + + /// With no Stop, a network-paused row stays for the scheduler. + #[test] + fn a_network_paused_run_no_stop_reached_stays_paused() { + let run = network_paused_run(); + + assert!(!run.deregister(&mock_app())); + + let row = run.row(); + assert_eq!(row.status, SessionStatus::Paused); + assert_eq!(row.completion_reason, Some(CompletionReason::RateLimited)); + assert_eq!(row.rate_limit_kind.as_deref(), Some("network")); + assert_eq!( + run.store.get_rate_limited_paused_sessions().unwrap().len(), + 1 + ); + } + /// The ordinary pause: no Stop reached the entry, so the row is left for /// the scheduler and the caller goes on to announce `paused`. #[test] diff --git a/apps/staged/src-tauri/src/session_runner/network_failure.rs b/apps/staged/src-tauri/src/session_runner/network_failure.rs new file mode 100644 index 000000000..0408a6b8a --- /dev/null +++ b/apps/staged/src-tauri/src/session_runner/network_failure.rs @@ -0,0 +1,508 @@ +//! Was a failed prompt caused by the network? +//! +//! A turn that fails because the agent could not reach its model provider is +//! worth parking until the network returns; every other failure is not. This +//! module answers that one question from the structured ACP error stored at +//! `provider_metadata.acp.promptError` and the session's provider. It is a +//! pure function: it reads nothing but its arguments and decides nothing +//! about what happens next. +//! +//! Structured fields exclude every failure class that is not the network +//! (wrong code, a lost remote transport, a rate limit, a billing or sign-in +//! error). No version of the protocol or of the bridges marks a failure as +//! network-caused on the failed prompt itself, though, so the last split +//! rests on the error message. That fallback is confined to +//! [`matched_fallback_signal`] below. **No other file may match error prose +//! to decide whether a failure was the network.** + +/// The JSON-RPC "internal error" code. Both bridges report a failed provider +/// request with it; `-32000` (authentication required) and every other code +/// are not network failures. +const INTERNAL_ERROR_CODE: i64 = -32603; + +/// `data.reason` on the error Staged's own driver writes when a remote +/// session's transport closes. That loss has its own recovery. +const INCOMING_TRANSPORT_CLOSED_REASON: &str = "incoming_transport_closed"; + +/// The `errorKind` values the Claude bridge can put on a network failure. +/// The CLI files connection errors, timeouts and real HTTP 5xx responses +/// alike under `server_error`, and a gateway's HTML page under `unknown`. +/// Every other kind, and a missing kind, is not a network failure. +const CLAUDE_NETWORK_ERROR_KINDS: &[&str] = &["server_error", "unknown"]; + +/// `data.reason` Goose puts on every failed turn, network or not. +const GOOSE_TURN_FAILED_REASON: &str = "turn_failed"; + +/// The signal that identified a failed prompt as a network failure, for logs +/// and metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NetworkFailure { + /// Claude: the CLI printed this errno token from its allow-list. + ClaudeErrno(&'static str), + /// Claude: the CLI's request timeout, which carries no token. + ClaudeRequestTimeout, + /// Claude: a proxy or gateway answered HTTP 200 with a page that is not + /// an API response. + ClaudeGatewayPage, + /// Goose: its `NetworkError` variant. + GooseNetworkError, +} + +impl NetworkFailure { + /// A stable name for the matched signal. + pub(crate) fn signal(&self) -> &'static str { + match self { + Self::ClaudeErrno(token) => token, + Self::ClaudeRequestTimeout => "request_timed_out", + Self::ClaudeGatewayPage => "gateway_page", + Self::GooseNetworkError => "goose_network_error", + } + } +} + +/// Classify a failed run as a network failure, or `None` for anything else. +/// +/// `prompt_failed` is the driver's own flag for the run +/// (`acp_client::SessionSettled::prompt_failed`) and `provider_metadata` the +/// session's stored metadata. The checks run in this order, and only the +/// last one reads error prose: +/// +/// 1. The run failed on the prompt: the agent answered `session/prompt` with +/// an error over a live transport. `promptError` can be left on the row by +/// an earlier event of the same turn, so a failure during setup, after a +/// cancel or an interrupt, or on a lost transport must not be judged by +/// it. The driver reports this as a flag, so no error text is parsed. +/// 2. The error code is `-32603`. +/// 3. `data.reason` is not `incoming_transport_closed`. +/// 4. The agent's own structured field allows a network cause. +/// 5. A row of the fallback table matches for that agent. +/// +/// A rate limit never reaches step 5: its `errorKind` fails step 4. Callers +/// still run the rate-limit check first. +pub(crate) fn classify_network_failure( + provider: Option<&str>, + prompt_failed: bool, + provider_metadata: Option<&serde_json::Value>, +) -> Option { + let agent = NetworkAgent::from_provider(provider?)?; + if !prompt_failed { + return None; + } + let prompt_error = provider_metadata?.pointer("/acp/promptError")?; + if prompt_error.get("code").and_then(|code| code.as_i64()) != Some(INTERNAL_ERROR_CODE) { + return None; + } + let data = prompt_error.get("data"); + let data_str = |field: &str| { + data.and_then(|data| data.get(field)) + .and_then(|value| value.as_str()) + }; + if data_str("reason") == Some(INCOMING_TRANSPORT_CLOSED_REASON) { + return None; + } + let structured_gate_passes = match agent { + NetworkAgent::Claude => { + data_str("errorKind").is_some_and(|kind| CLAUDE_NETWORK_ERROR_KINDS.contains(&kind)) + } + NetworkAgent::Goose => data_str("reason") == Some(GOOSE_TURN_FAILED_REASON), + }; + if !structured_gate_passes { + return None; + } + let message = prompt_error.get("message").and_then(|m| m.as_str())?; + matched_fallback_signal(agent, message) +} + +/// The agents whose failed prompts can be classified. Codex is absent on +/// purpose: it retries a failed connection inside the turn without limit, and +/// its other failures end the turn as text, not as an error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NetworkAgent { + Claude, + Goose, +} + +impl NetworkAgent { + fn from_provider(provider: &str) -> Option { + match provider { + "claude" => Some(Self::Claude), + "goose" => Some(Self::Goose), + _ => None, + } + } +} + +// --------------------------------------------------------------------------- +// The fallback table: the only place that reads error prose. +// +// Each row applies only after the structured checks in +// `classify_network_failure` have passed, and only for its own agent. Each +// exists because the agent forwards no structured cause for that failure. +// When the Claude bridge starts copying the SDK's `api_error_status` into the +// prompt error's `data`, check that field ahead of this table and delete the +// Claude rows it covers. +// --------------------------------------------------------------------------- + +/// Claude row 1: an errno token in parentheses at the end of the message. +/// +/// The CLI takes the raw `code` from the error chain and prints it in +/// parentheses, so the token is a machine string even though it sits inside +/// prose; the bridge forwards no cause field. Only these tokens count. TLS +/// certificate codes and `ERR_PROXY_TUNNEL` are left out because waiting for +/// the network does not fix them: they need the user. +const CLAUDE_NETWORK_ERRNO_TOKENS: &[&str] = &[ + "ENOTFOUND", + "EAI_AGAIN", + "FailedToOpenSocket", + "ECONNREFUSED", + "ConnectionRefused", + "ECONNRESET", + "EPIPE", + "ECONNABORTED", + "ConnectionClosed", + "ERR_SOCKET_CLOSED", + "UND_ERR_SOCKET", + "ENETUNREACH", + "ENETDOWN", + "EHOSTUNREACH", + "EHOSTDOWN", + "ETIMEDOUT", +]; + +/// The prefix the Claude bridge puts on every failed provider request. +const CLAUDE_INTERNAL_ERROR_PREFIX: &str = "Internal error: "; + +/// Claude row 2: the whole message after [`CLAUDE_INTERNAL_ERROR_PREFIX`]. +/// A fixed English constant in the CLI, which emits no token for a timeout. +const CLAUDE_REQUEST_TIMED_OUT: &str = "Request timed out"; + +/// Claude row 3: both must appear. A gateway or captive-portal page answered +/// in place of the API; the bridge has no kind for it beyond `unknown`. +const CLAUDE_GATEWAY_PAGE_MARKERS: [&str; 2] = ["(HTTP 200)", "empty or malformed response"]; + +/// Goose row: the `Display` prefix of Goose's internal `NetworkError` +/// variant. `data.reason` is `turn_failed` for every Goose failure, so +/// nothing structured separates this from a 5xx or a rate limit. +const GOOSE_NETWORK_ERROR_PREFIX: &str = "Network error: "; + +fn matched_fallback_signal(agent: NetworkAgent, message: &str) -> Option { + let message = message.trim(); + match agent { + NetworkAgent::Claude => { + if let Some(token) = trailing_parenthesized_token(message) + .and_then(|token| CLAUDE_NETWORK_ERRNO_TOKENS.iter().find(|t| **t == token)) + { + return Some(NetworkFailure::ClaudeErrno(token)); + } + if message.strip_prefix(CLAUDE_INTERNAL_ERROR_PREFIX) == Some(CLAUDE_REQUEST_TIMED_OUT) + { + return Some(NetworkFailure::ClaudeRequestTimeout); + } + if CLAUDE_GATEWAY_PAGE_MARKERS + .iter() + .all(|marker| message.contains(marker)) + { + return Some(NetworkFailure::ClaudeGatewayPage); + } + None + } + NetworkAgent::Goose => message + .starts_with(GOOSE_NETWORK_ERROR_PREFIX) + .then_some(NetworkFailure::GooseNetworkError), + } +} + +/// The text inside a `(...)` that ends the message, if the message ends +/// with one. +fn trailing_parenthesized_token(message: &str) -> Option<&str> { + let inner = message.strip_suffix(')')?; + let open = inner.rfind('(')?; + Some(&inner[open + 1..]) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const DNS_MESSAGE: &str = "Internal error: API Error: Can't reach the API server — check your internet or DNS (ENOTFOUND)"; + const REFUSED_MESSAGE: &str = "Internal error: API Error: Connection refused — a firewall or proxy may be blocking it (ECONNREFUSED)"; + const TIMEOUT_MESSAGE: &str = "Internal error: Request timed out"; + const GATEWAY_MESSAGE: &str = "Internal error: API Error: API returned an empty or malformed response (HTTP 200) — check for a proxy or gateway intercepting the request. Response: content-type html, body is an HTML page, 62825 bytes, request-id absent, server cloudflare, intermediary headers cf-ray content-encoding transfer-encoding. This was the non-streaming retry of streaming request (no Anthropic request-id), which failed with: no_events, StreamNoEventsError; 0 stream events received."; + + fn metadata(prompt_error: serde_json::Value) -> serde_json::Value { + json!({ "acp": { "promptError": prompt_error } }) + } + + fn claude_error(kind: &str, message: &str) -> serde_json::Value { + metadata(json!({ "code": -32603, "message": message, "data": { "errorKind": kind } })) + } + + fn goose_error(message: &str) -> serde_json::Value { + metadata(json!({ "code": -32603, "message": message, "data": { "reason": "turn_failed" } })) + } + + fn classify(provider: &str, metadata: &serde_json::Value) -> Option { + classify_network_failure(Some(provider), true, Some(metadata)) + } + + #[test] + fn the_four_stored_claude_network_payloads_classify_as_network() { + assert_eq!( + classify("claude", &claude_error("server_error", DNS_MESSAGE)), + Some(NetworkFailure::ClaudeErrno("ENOTFOUND")) + ); + assert_eq!( + classify("claude", &claude_error("server_error", REFUSED_MESSAGE)), + Some(NetworkFailure::ClaudeErrno("ECONNREFUSED")) + ); + // The stored timeout row carries `unknown`; `server_error` is what + // the CLI assigns today. + for kind in ["unknown", "server_error"] { + assert_eq!( + classify("claude", &claude_error(kind, TIMEOUT_MESSAGE)), + Some(NetworkFailure::ClaudeRequestTimeout), + "{kind}" + ); + } + assert_eq!( + classify("claude", &claude_error("unknown", GATEWAY_MESSAGE)), + Some(NetworkFailure::ClaudeGatewayPage) + ); + } + + #[test] + fn every_allow_listed_errno_token_matches_and_names_itself() { + for token in CLAUDE_NETWORK_ERRNO_TOKENS { + let message = format!("Internal error: API Error: Connection error ({token})"); + let failure = classify("claude", &claude_error("server_error", &message)) + .unwrap_or_else(|| panic!("{token} should classify")); + assert_eq!(failure, NetworkFailure::ClaudeErrno(token)); + assert_eq!(failure.signal(), *token); + } + } + + #[test] + fn a_real_5xx_without_a_token_is_not_a_network_failure() { + for message in [ + "Internal error: API Error: 500 Internal server error", + "Internal error: API Error: Repeated 529 Overloaded errors", + // A token that is not at the end, or not in parentheses, is prose. + "Internal error: API Error: upstream said ECONNRESET while proxying", + "Internal error: API Error: (ECONNRESET) reported by upstream, HTTP 502", + // Not the whole message. + "Internal error: Request timed out while waiting for tool output", + "Internal error: API Error: Request timed out", + // One gateway marker without the other. + "Internal error: API Error: unexpected body (HTTP 200)", + ] { + assert_eq!( + classify("claude", &claude_error("server_error", message)), + None, + "{message}" + ); + } + } + + #[test] + fn a_tls_or_proxy_tunnel_code_needs_the_user_and_is_not_paused() { + for token in [ + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "SELF_SIGNED_CERT_IN_CHAIN", + "CERT_HAS_EXPIRED", + "ERR_PROXY_TUNNEL", + ] { + let message = format!("Internal error: API Error: Connection error ({token})"); + assert_eq!( + classify("claude", &claude_error("server_error", &message)), + None, + "{token}" + ); + } + } + + #[test] + fn every_other_claude_error_kind_is_excluded_even_with_a_network_message() { + for kind in [ + "authentication_failed", + "oauth_org_not_allowed", + "account_on_hold", + "verification_required", + "billing_error", + "rate_limit", + "overloaded", + "invalid_request", + "model_not_found", + "max_output_tokens", + "cloud_credential_error", + "incomplete_tool_call", + "no_result", + "worker_shutdown", + "transport_lost", + ] { + assert_eq!( + classify("claude", &claude_error(kind, DNS_MESSAGE)), + None, + "{kind}" + ); + } + } + + #[test] + fn a_missing_or_non_string_claude_error_kind_is_excluded() { + for data in [ + json!(null), + json!({}), + json!({ "errorKind": 7 }), + json!("server_error"), + ] { + let metadata = metadata(json!({ + "code": -32603, + "message": DNS_MESSAGE, + "data": data, + })); + assert_eq!(classify("claude", &metadata), None, "{data}"); + } + let no_data = metadata(json!({ "code": -32603, "message": TIMEOUT_MESSAGE })); + assert_eq!(classify("claude", &no_data), None); + } + + #[test] + fn a_rate_limit_is_never_a_network_failure() { + let metadata = metadata(json!({ + "code": -32603, + "message": "Internal error: API Error: Rate limited (ETIMEDOUT)", + "data": { "errorKind": "rate_limit", "rateLimit": { "resetsAt": 1_800_000_000 } }, + })); + assert_eq!(classify("claude", &metadata), None); + } + + #[test] + fn any_code_other_than_internal_error_is_excluded() { + let auth = metadata(json!({ "code": -32000, "message": "Authentication required" })); + assert_eq!(classify("claude", &auth), None); + for code in [ + json!(-32000), + json!(-32602), + json!(-32002), + json!("-32603"), + json!(null), + ] { + let metadata = metadata(json!({ + "code": code, + "message": DNS_MESSAGE, + "data": { "errorKind": "server_error" }, + })); + assert_eq!(classify("claude", &metadata), None, "{code}"); + } + } + + #[test] + fn a_lost_remote_transport_is_excluded() { + // As stored: written by Staged's own driver, with no error kind. + let stored = metadata(json!({ + "code": -32603, + "message": "Incoming transport closed", + "data": { "reason": "incoming_transport_closed", "method": "session/prompt" }, + })); + // Even if a kind and a network message rode along with it. + let with_kind = metadata(json!({ + "code": -32603, + "message": DNS_MESSAGE, + "data": { "reason": "incoming_transport_closed", "errorKind": "server_error" }, + })); + for provider in ["claude", "goose"] { + assert_eq!(classify(provider, &stored), None, "{provider}"); + assert_eq!(classify(provider, &with_kind), None, "{provider}"); + } + } + + /// A `promptError` left on the row by an earlier event says nothing + /// about a run the driver did not report as a failed prompt, whatever + /// the stored error looks like. + #[test] + fn a_stale_prompt_error_is_ignored_when_the_run_did_not_fail_on_the_prompt() { + for (provider, metadata) in [ + ("claude", claude_error("server_error", DNS_MESSAGE)), + ("goose", goose_error("Network error: connection reset")), + ] { + assert!(classify(provider, &metadata).is_some()); + assert_eq!( + classify_network_failure(Some(provider), false, Some(&metadata)), + None, + "{provider}" + ); + } + } + + #[test] + fn a_prompt_failure_without_a_stored_prompt_error_is_not_classified() { + assert_eq!(classify_network_failure(Some("claude"), true, None), None); + assert_eq!( + classify_network_failure(Some("claude"), true, Some(&json!({ "acp": {} }))), + None + ); + } + + #[test] + fn goose_network_errors_classify_and_goose_server_errors_do_not() { + let network = goose_error( + "Network error: error sending request for url (https://api.example.com/v1/messages)", + ); + let failure = classify("goose", &network); + assert_eq!(failure, Some(NetworkFailure::GooseNetworkError)); + assert_eq!(failure.unwrap().signal(), "goose_network_error"); + + for message in [ + "Server error: 503 Service Unavailable", + "Rate limit exceeded: Network error: try later", + "Request failed: Network error: mentioned later (ENOTFOUND)", + ] { + assert_eq!(classify("goose", &goose_error(message)), None, "{message}"); + } + } + + #[test] + fn goose_needs_its_own_turn_failed_reason() { + for data in [ + json!({ "reason": "credits_exhausted" }), + json!({ "errorKind": "server_error" }), + json!(null), + ] { + let metadata = metadata(json!({ + "code": -32603, + "message": "Network error: connection reset", + "data": data, + })); + assert_eq!(classify("goose", &metadata), None, "{data}"); + } + } + + #[test] + fn each_fallback_row_applies_only_to_its_own_agent() { + // Goose's prefix under Claude's gate, and Claude's rows under Goose's. + assert_eq!( + classify( + "claude", + &claude_error("server_error", "Network error: connection reset") + ), + None + ); + for message in [DNS_MESSAGE, TIMEOUT_MESSAGE, GATEWAY_MESSAGE] { + assert_eq!(classify("goose", &goose_error(message)), None, "{message}"); + } + } + + #[test] + fn codex_and_unknown_providers_are_never_classified() { + let mut metadata = claude_error("server_error", DNS_MESSAGE); + metadata["acp"]["promptError"]["data"]["reason"] = json!("turn_failed"); + for provider in [Some("codex"), Some("amp"), Some(""), None] { + assert_eq!( + classify_network_failure(provider, true, Some(&metadata)), + None, + "{provider:?}" + ); + } + } +} diff --git a/apps/staged/src-tauri/src/store/messages.rs b/apps/staged/src-tauri/src/store/messages.rs index 7922b6358..2645a71f3 100644 --- a/apps/staged/src-tauri/src/store/messages.rs +++ b/apps/staged/src-tauri/src/store/messages.rs @@ -18,6 +18,16 @@ const VISIBLE_MESSAGE_FILTER: &str = "NOT (content = '' AND acp_event_kind IS NO /// button compares against. pub(super) const VISIBLE_ASSISTANT_FILTER: &str = "role = 'assistant' AND NOT (content = '' AND acp_event_kind IS NOT NULL)"; +/// A row of the agent's own output for a turn: text, a tool call or its +/// result, a plan, or the turn's response. Hidden rows tagged with an +/// `acp_origin` and the rows a connection writes before it sends the prompt +/// are left out. +const AGENT_OUTPUT_FILTER: &str = "acp_origin IS NULL + AND (role IN ('tool_call', 'tool_result') + OR (role = 'assistant' + AND (acp_event_kind IS NULL + OR acp_event_kind IN ('agent_message_chunk', 'plan_update', + 'response_result', 'prompt_response'))))"; /// ACP rows the transcript needs on top of [`Store::get_session_messages`]: /// hidden rows of the kinds the frontend reads (`acpTranscript.ts`, /// `acpConfigMetadata.ts`), plus visible tool rows still in flight — the @@ -676,18 +686,50 @@ impl Store { &self, session_id: &str, message_id: i64, + ) -> Result { + let conn = self.conn.lock().unwrap(); + conn.query_row( + &format!( + "SELECT EXISTS( + SELECT 1 FROM session_messages + WHERE session_id = ?1 AND id > ?2 AND {AGENT_OUTPUT_FILTER})" + ), + params![session_id, message_id], + |row| row.get(0), + ) + .map_err(Into::into) + } + + /// Whether a session's agent reached its model provider at or after + /// `since`: the session recorded a tool call or its result, a result the + /// agent did not mark as an error, or a prompt response. + /// + /// Backs the network gate's probe check, which knows when it claimed the + /// probe but not which row its prompt will be. It is narrower than + /// [`Self::agent_answered_message`] on purpose. A turn that fails because + /// the provider is unreachable still records agent output: Claude prints + /// the failure as a message chunk ("API Error: Can't reach the API + /// server"), then a `response_result` with `is_error: true`, and the + /// writer stores that text as a plain assistant row. None of those shows + /// that a request got through, so message text and plans never count + /// here and a result counts only without the error mark. + pub fn session_reached_provider_since( + &self, + session_id: &str, + since: i64, ) -> Result { let conn = self.conn.lock().unwrap(); conn.query_row( "SELECT EXISTS( SELECT 1 FROM session_messages - WHERE session_id = ?1 AND id > ?2 AND acp_origin IS NULL + WHERE session_id = ?1 AND created_at >= ?2 AND acp_origin IS NULL AND (role IN ('tool_call', 'tool_result') - OR (role = 'assistant' - AND (acp_event_kind IS NULL - OR acp_event_kind IN ('agent_message_chunk', 'plan_update', - 'response_result', 'prompt_response')))))", - params![session_id, message_id], + OR (role = 'assistant' AND acp_event_kind = 'prompt_response') + OR (role = 'assistant' AND acp_event_kind = 'response_result' + AND CASE WHEN json_valid(acp_content) + THEN json_extract(acp_content, '$.is_error') = 0 + ELSE 0 END)))", + params![session_id, since], |row| row.get(0), ) .map_err(Into::into) diff --git a/apps/staged/src-tauri/src/store/models.rs b/apps/staged/src-tauri/src/store/models.rs index c7bc510a6..911787eae 100644 --- a/apps/staged/src-tauri/src/store/models.rs +++ b/apps/staged/src-tauri/src/store/models.rs @@ -574,7 +574,9 @@ pub enum CompletionReason { /// output ended, or bloxlet reports it stopped — e.g. bloxlet restarted /// under it. The error message names the cause. RemoteAgentLost, - /// Agent stopped because the provider reported a rate or usage limit. + /// Paused for an automatic resume. The provider reported a rate or usage + /// limit, or the prompt failed on the network: `rate_limit_kind` is + /// `"network"` for the latter. The name predates the network case. RateLimited, /// Staged closed while the session was still running. AppQuit, diff --git a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts index 6e2c1f4d5..3ff9e7acd 100644 --- a/apps/staged/src/lib/features/projects/projectNoteRow.test.ts +++ b/apps/staged/src/lib/features/projects/projectNoteRow.test.ts @@ -98,6 +98,31 @@ describe('projectNoteRow', () => { expect(failed(null).title).toBe('Session finished — no note created'); }); + it('titles a paused note by its kind of pause', () => { + const paused = (overrides: Partial) => + projectNoteRow( + note({ + sessionStatus: 'paused', + content: '', + title: '', + completionReason: 'rate_limited', + ...overrides, + }), + { deleting: false, nowMs: NOW } + ); + const resetsAt = '2026-06-24T15:45:00Z'; + const resetTime = new Date(resetsAt).toLocaleTimeString([], { + hour: 'numeric', + minute: '2-digit', + }); + expect(paused({ rateLimitKind: 'network' }).title).toBe('Paused: network unavailable.'); + expect(paused({}).title).toBe('Paused by API limit — no note yet'); + // The reset time dates the pause; it is not the session's error message. + expect(paused({ rateLimitResetsAt: resetsAt }).title).toBe( + `Paused by API limit — resets around ${resetTime}` + ); + }); + it('subtitles a deleting row "Deleting..." whatever its phase', () => { const live = { deleting: true, nowMs: NOW, liveHint: 'Reading' }; expect(projectNoteRow(note(), live).secondaryMeta).toBe('Deleting...'); diff --git a/apps/staged/src/lib/features/projects/projectNoteRow.ts b/apps/staged/src/lib/features/projects/projectNoteRow.ts index 084350e50..6ecd339d7 100644 --- a/apps/staged/src/lib/features/projects/projectNoteRow.ts +++ b/apps/staged/src/lib/features/projects/projectNoteRow.ts @@ -29,6 +29,7 @@ export type ProjectNoteRowSource = Pick< | 'updatedAt' | 'completedAt' | 'rateLimitResetsAt' + | 'rateLimitKind' >; export type ProjectNoteRowLive = { @@ -62,7 +63,10 @@ export function projectNoteRow( case 'failed': row = { type: 'failed-note', - title: failedArtifactSubtitle(note.completionReason, 'note', note.rateLimitResetsAt), + title: failedArtifactSubtitle(note.completionReason, 'note', { + rateLimitResetsAt: note.rateLimitResetsAt, + rateLimitKind: note.rateLimitKind, + }), secondaryMeta: undefined, }; break; diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts index 7907bcff2..b5abed360 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.test.ts @@ -421,6 +421,77 @@ describe('newestUserMessageId', () => { 'This session paused because the provider reported an API limit. Resume manually when ready.' ); }); + + it('keeps the API limit copy for an API-limit kind', () => { + expect( + sessionEndMessage({ completionReason: 'rate_limited', rateLimitKind: 'five_hour' }) + ).toBe('This session paused because the provider reported an API limit.'); + }); + + it('explains a network pause', () => { + expect(sessionEndMessage({ completionReason: 'rate_limited', rateLimitKind: 'network' })).toBe( + 'Paused: network unavailable. Resumes when the connection returns.' + ); + }); + + it('gives no retry time for a network pause waiting behind the gate', () => { + // As the scheduler writes it: `nextRetryAt` is when the row paused, so + // it is already in the past and stays there for the whole outage. + const pausedAt = Date.UTC(2026, 5, 24, 15, 45); + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { + staged: { + rateLimitAutoRecovery: { + attempt: 0, + maxAttempts: 8, + nextRetryAt: pausedAt, + reason: 'network_wait', + }, + }, + }, + }) + ).toBe('Paused: network unavailable. Resumes when the connection returns.'); + }); + + it('says when a network pause on its own backoff tries again', () => { + const nextRetryAt = Date.UTC(2026, 5, 24, 15, 45); + const retryTime = new Date(nextRetryAt).toLocaleTimeString([], { + hour: 'numeric', + minute: '2-digit', + }); + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { + staged: { + rateLimitAutoRecovery: { attempt: 2, maxAttempts: 8, nextRetryAt, reason: 'backoff' }, + }, + }, + }) + ).toBe( + `Paused: network unavailable. Resumes when the connection returns. Trying again around ${retryTime}.` + ); + }); + + it('asks for a manual resume of a network pause that will not resume by itself', () => { + for (const rateLimitAutoRecovery of [ + { attempt: 9, maxAttempts: 8, gaveUpAt: Date.UTC(2026, 5, 24, 15, 45) }, + { attempt: 0, gaveUpAt: Date.UTC(2026, 5, 24, 15, 45), reason: 'probe_limit_exceeded' }, + { unsafeToAutoResume: true, reason: 'pending_tool_call' }, + ]) { + expect( + sessionEndMessage({ + completionReason: 'rate_limited', + rateLimitKind: 'network', + providerMetadata: { staged: { rateLimitAutoRecovery } }, + }) + ).toBe('Paused: network unavailable. Resume manually when ready.'); + } + }); }); describe('detached remote note sessions', () => { diff --git a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts index 7269551f5..cb9b8c084 100644 --- a/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts +++ b/apps/staged/src/lib/features/sessions/sessionModalHelpers.ts @@ -8,7 +8,13 @@ import { RECONCILE_WORKSTATION_GONE, isReconcileCancelNotice, } from '../../shared/disconnectedSession'; -import { formatRateLimitResetTime } from '../timeline/sessionFailureCopy'; +import { + NETWORK_PAUSED, + NETWORK_PAUSED_RESUMES, + NETWORK_WAIT_REASON, + formatRateLimitResetTime, + isNetworkPause, +} from '../timeline/sessionFailureCopy'; /** A text snippet attached to a new-session prompt (modal-local, never persisted). */ export interface TextSnippet { @@ -165,7 +171,9 @@ export function sessionEndNoticeKind( */ export function sessionEndMessage( current: Pick & - Partial> + Partial< + Pick + > ): string { switch (current.errorMessage) { case RECONCILE_TURN_UNDETERMINED: @@ -225,7 +233,7 @@ function detachedEndMessage(errorMessage: string | null | undefined): string { function endReasonMessage( current: Pick & - Partial> + Partial> ): string { switch (current.completionReason) { case 'crashed': @@ -254,12 +262,14 @@ function endReasonMessage( /** * Why a rate-limited session paused and what happens next: the scheduler's * bookkeeping in `providerMetadata.staged.rateLimitAutoRecovery` when it has - * any, else the provider's reset time. + * any, else the provider's reset time. A network pause is the same pause + * under another kind, and has its own copy. */ function rateLimitEndMessage( - current: Partial> + current: Partial> ): string { const recovery = rateLimitAutoRecovery(current.providerMetadata); + if (isNetworkPause(current.rateLimitKind)) return networkPauseEndMessage(recovery); if (recovery?.gaveUpAt) { return `This session paused because the provider reported an API limit. Auto-retry gave up after ${recovery.attempt ?? 'several'} attempts.`; } @@ -276,6 +286,25 @@ function rateLimitEndMessage( return 'This session paused because the provider reported an API limit.'; } +/** + * The notice for a network pause. A row the scheduler gave up on, or will not + * resume by itself, no longer resumes when the connection returns, so it says + * what to do instead of repeating that promise. + * + * A row waiting behind the scheduler's network gate gets no retry time: its + * `nextRetryAt` is the time it paused, and the gate's next probe time is not + * exposed. Only a pause on the session's own backoff has a time to show. + */ +function networkPauseEndMessage(recovery: RateLimitAutoRecovery | null): string { + if (recovery?.gaveUpAt || recovery?.unsafeToAutoResume) { + return `${NETWORK_PAUSED} Resume manually when ready.`; + } + if (recovery?.nextRetryAt && recovery.reason !== NETWORK_WAIT_REASON) { + return `${NETWORK_PAUSED_RESUMES} Trying again around ${formatTimestamp(recovery.nextRetryAt)}.`; + } + return NETWORK_PAUSED_RESUMES; +} + /** * A failed session's error, as the chat pane shows it. Reconcile's terse * "workstation no longer exists" is reframed around what it means for the @@ -321,6 +350,7 @@ interface RateLimitAutoRecovery { nextRetryAt?: number | null; gaveUpAt?: number | null; unsafeToAutoResume?: boolean; + reason?: string | null; } function rateLimitAutoRecovery( diff --git a/apps/staged/src/lib/features/timeline/BranchTimeline.svelte b/apps/staged/src/lib/features/timeline/BranchTimeline.svelte index f75a351c0..795798cc8 100644 --- a/apps/staged/src/lib/features/timeline/BranchTimeline.svelte +++ b/apps/staged/src/lib/features/timeline/BranchTimeline.svelte @@ -766,12 +766,11 @@ if (isFailed) { phase = 'failed'; type = 'failed-commit'; - secondaryMeta = failedArtifactSubtitle( - commit.completionReason, - 'commit', - commit.sessionErrorMessage, - commit.rateLimitResetsAt - ); + secondaryMeta = failedArtifactSubtitle(commit.completionReason, 'commit', { + errorMessage: commit.sessionErrorMessage, + rateLimitResetsAt: commit.rateLimitResetsAt, + rateLimitKind: commit.rateLimitKind, + }); } else if (isQueued) { phase = 'queued'; type = 'queued-commit'; @@ -840,13 +839,12 @@ if (phase === 'failed') { type = 'failed-note'; - secondaryMeta = failedArtifactSubtitle( - note.completionReason, - 'note', - note.sessionErrorMessage, - note.rateLimitResetsAt, - note.isPollingForNote - ); + secondaryMeta = failedArtifactSubtitle(note.completionReason, 'note', { + errorMessage: note.sessionErrorMessage, + rateLimitResetsAt: note.rateLimitResetsAt, + rateLimitKind: note.rateLimitKind, + isPollingForNote: note.isPollingForNote, + }); } else if (phase === 'queued') { type = 'queued-note'; secondaryMeta = queuedMeta; @@ -911,12 +909,11 @@ if (isFailed) { phase = 'failed'; type = 'failed-review'; - meta = failedArtifactSubtitle( - review.completionReason, - 'review', - review.sessionErrorMessage, - review.rateLimitResetsAt - ); + meta = failedArtifactSubtitle(review.completionReason, 'review', { + errorMessage: review.sessionErrorMessage, + rateLimitResetsAt: review.rateLimitResetsAt, + rateLimitKind: review.rateLimitKind, + }); } else if (isQueued) { phase = 'queued'; type = 'queued-review'; diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts index ca91345fb..4ef36ae09 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.test.ts @@ -20,10 +20,10 @@ describe('failedArtifactSubtitle', () => { }); it('says "yet" for a detached remote session only while Staged is still working on its note', () => { - expect(failedArtifactSubtitle('detached', 'note', null, null, true)).toBe( + expect(failedArtifactSubtitle('detached', 'note', { isPollingForNote: true })).toBe( 'Lost the remote session — no note created yet' ); - expect(failedArtifactSubtitle('detached', 'note', null, null, false)).toBe( + expect(failedArtifactSubtitle('detached', 'note', { isPollingForNote: false })).toBe( 'Lost the remote session — no note created' ); expect(failedArtifactSubtitle('detached', 'note')).toBe( @@ -47,12 +47,12 @@ describe('failedArtifactSubtitle', () => { }); it('says a deleted workstation, not a crash, ended the session', () => { - expect(failedArtifactSubtitle('crashed', 'commit', RECONCILE_WORKSTATION_GONE)).toBe( - 'Workstation deleted — no commit created' - ); - expect(failedArtifactSubtitle('crashed', 'review', RECONCILE_WORKSTATION_GONE)).toBe( - 'Workstation deleted — no comments created' - ); + expect( + failedArtifactSubtitle('crashed', 'commit', { errorMessage: RECONCILE_WORKSTATION_GONE }) + ).toBe('Workstation deleted — no commit created'); + expect( + failedArtifactSubtitle('crashed', 'review', { errorMessage: RECONCILE_WORKSTATION_GONE }) + ).toBe('Workstation deleted — no comments created'); }); it('reads a turn Staged couldn’t follow as an interruption, not a Stop', () => { @@ -61,22 +61,24 @@ describe('failedArtifactSubtitle', () => { RECONCILE_NO_WORKSPACE, RECONCILE_PROMPT_NOT_RECEIVED, ]) { - expect(failedArtifactSubtitle('interrupted', 'note', message)).toBe( + expect(failedArtifactSubtitle('interrupted', 'note', { errorMessage: message })).toBe( 'Session interrupted — no note created' ); } }); it('keeps a Stop the workstation didn’t confirm a Stop', () => { - expect(failedArtifactSubtitle('interrupted', 'commit', RECONCILE_MAY_STILL_RUN)).toBe( - 'Session stopped — no commit created' - ); + expect( + failedArtifactSubtitle('interrupted', 'commit', { errorMessage: RECONCILE_MAY_STILL_RUN }) + ).toBe('Session stopped — no commit created'); }); it('falls back to the reason for any other message', () => { - expect(failedArtifactSubtitle('crashed', 'commit', 'Blox process exited with code 1')).toBe( - 'Session crashed — no commit created' - ); + expect( + failedArtifactSubtitle('crashed', 'commit', { + errorMessage: 'Blox process exited with code 1', + }) + ).toBe('Session crashed — no commit created'); }); it('explains API limit failures', () => { @@ -89,8 +91,44 @@ describe('failedArtifactSubtitle', () => { const resetTime = formatRateLimitResetTime('2026-06-24T15:45:00Z'); expect(resetTime).toBeTruthy(); - expect(failedArtifactSubtitle('rate_limited', 'commit', null, '2026-06-24T15:45:00Z')).toBe( - `Paused by API limit — resets around ${resetTime}` + expect( + failedArtifactSubtitle('rate_limited', 'commit', { + rateLimitResetsAt: '2026-06-24T15:45:00Z', + }) + ).toBe(`Paused by API limit — resets around ${resetTime}`); + }); + + it('keeps the API limit copy for an API-limit kind', () => { + expect(failedArtifactSubtitle('rate_limited', 'note', { rateLimitKind: 'five_hour' })).toBe( + 'Paused by API limit — no note yet' + ); + expect(failedArtifactSubtitle('rate_limited', 'note', { rateLimitKind: null })).toBe( + 'Paused by API limit — no note yet' + ); + }); + + // The row cannot know whether the scheduler gave up on the pause or will + // not resume it by itself, so it must not say the session resumes. + it('names a network pause without promising a resume, whatever the artifact', () => { + for (const kind of ['commit', 'note', 'review'] as const) { + const subtitle = failedArtifactSubtitle('rate_limited', kind, { rateLimitKind: 'network' }); + expect(subtitle).toBe('Paused: network unavailable.'); + expect(subtitle).not.toMatch(/resume/i); + } + }); + + it('does not date a network pause by a provider reset time', () => { + expect( + failedArtifactSubtitle('rate_limited', 'commit', { + rateLimitKind: 'network', + rateLimitResetsAt: '2026-06-24T15:45:00Z', + }) + ).toBe('Paused: network unavailable.'); + }); + + it('reads the network kind only on a rate-limit pause', () => { + expect(failedArtifactSubtitle('crashed', 'commit', { rateLimitKind: 'network' })).toBe( + 'Session crashed — no commit created' ); }); }); diff --git a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts index 57388235b..429053632 100644 --- a/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts +++ b/apps/staged/src/lib/features/timeline/sessionFailureCopy.ts @@ -24,19 +24,59 @@ const RECONCILE_INTERRUPTIONS: ReadonlySet = new Set([ RECONCILE_PROMPT_NOT_RECEIVED, ]); +/** + * The `rateLimitKind` of a `rate_limited` pause whose turn failed because the + * agent could not reach its model provider, not because of an API limit + * (`NETWORK_PAUSE_KIND` in `src-tauri/src/rate_limit_scheduler.rs`). + */ +export const NETWORK_PAUSE_KIND = 'network'; + +/** + * The `reason` in a network pause's retry bookkeeping while it waits behind + * the scheduler's network gate (`NETWORK_WAIT_REASON` in + * `src-tauri/src/rate_limit_scheduler.rs`). Such a row's `nextRetryAt` is the + * time it paused, not the time of its next attempt. + */ +export const NETWORK_WAIT_REASON = 'network_wait'; + +/** What a network pause says wherever it is named: the row subtitle and the chat alert. */ +export const NETWORK_PAUSED = 'Paused: network unavailable.'; +/** + * The same, with the promise of an automatic resume. Only for a caller that + * has the scheduler's bookkeeping and so knows the promise holds. + */ +export const NETWORK_PAUSED_RESUMES = `${NETWORK_PAUSED} Resumes when the connection returns.`; + +/** Whether a `rate_limited` pause with this `rateLimitKind` is a network pause. */ +export function isNetworkPause(rateLimitKind: string | null | undefined): boolean { + return rateLimitKind === NETWORK_PAUSE_KIND; +} + +/** The session fields `failedArtifactSubtitle` reads beyond the completion reason. */ +export interface FailedArtifactDetails { + /** The session's error: reconcile's messages say more than its completion reason does. */ + errorMessage?: string | null; + /** Dates an API-limit pause's subtitle. */ + rateLimitResetsAt?: string | null; + /** Tells a network pause from an API-limit pause. */ + rateLimitKind?: string | null; + /** The note row's `NoteTimelineItem.isPollingForNote`. */ + isPollingForNote?: boolean; +} + /** * The subtitle of a timeline row whose session ended without producing its - * artifact. `errorMessage` is the session's: reconcile's messages say more - * than its completion reason does. `rateLimitResetsAt` dates a rate-limit - * pause's subtitle. `isPollingForNote` is the note row's - * `NoteTimelineItem.isPollingForNote`. + * artifact. */ export function failedArtifactSubtitle( completionReason: string | null | undefined, kind: ArtifactKind, - errorMessage?: string | null, - rateLimitResetsAt?: string | null, - isPollingForNote = false + { + errorMessage, + rateLimitResetsAt, + rateLimitKind, + isPollingForNote = false, + }: FailedArtifactDetails = {} ): string { const noun = artifactNoun[kind]; if (errorMessage === RECONCILE_WORKSTATION_GONE) { @@ -51,6 +91,11 @@ export function failedArtifactSubtitle( case 'remote_agent_lost': return `Workspace agent lost — no ${noun} created`; case 'rate_limited': + // A timeline row does not carry the scheduler's bookkeeping, so it + // cannot tell a pause that will resume by itself from one the + // scheduler gave up on or will not resume (a pending tool call). It + // makes no promise either way; the chat alert, which can tell, does. + if (isNetworkPause(rateLimitKind)) return NETWORK_PAUSED; return rateLimitPausedSubtitle(rateLimitResetsAt) ?? `Paused by API limit — no ${noun} yet`; case 'app_quit': return `Session interrupted — no ${noun} created`; diff --git a/crates/acp-client/src/driver.rs b/crates/acp-client/src/driver.rs index e3e8afa93..2cf70da3b 100644 --- a/crates/acp-client/src/driver.rs +++ b/crates/acp-client/src/driver.rs @@ -1052,6 +1052,14 @@ pub struct SessionSettled { /// after a Stop cancel it again over a fresh connection /// ([`AcpDriver::attach_cancel`]). pub background_work_left: bool, + /// The agent answered the last turn's `session/prompt` with an error of + /// its own, over a transport that was still up to carry it. `false` for + /// a failure during setup, after a cancel or an interrupt, and for one + /// the SDK made because the transport closed. + /// + /// This is the structured form of "the run failed on the prompt": a + /// caller that needs to know it reads this instead of the error text. + pub prompt_failed: bool, } impl SessionSettled { @@ -3155,6 +3163,7 @@ impl AcpDriver { transport_loss, cancel_unacknowledged: transport.cancel_unacknowledged(), background_work_left: transport.background_work_left(), + prompt_failed: transport.answered_error(), }); if let Some(reply) = pending_reply.borrow_mut().take() { @@ -7774,6 +7783,12 @@ async fn record_prompt_error( .await; } +/// How the driver's error text begins when the agent answered `session/prompt` +/// with an error other than `auth_required` (see [`describe_prompt_error`]). +/// Display text only: a caller that needs to know a run failed on the prompt +/// reads [`SessionSettled::prompt_failed`]. +const PROMPT_FAILED_PREFIX: &str = "Prompt failed:"; + /// The text a failed `session/prompt` surfaces to the user. /// /// amp, claude, and goose with a native provider accept `session/new` while @@ -7795,7 +7810,7 @@ fn describe_prompt_error( required.log_methods(&format!("required to {operation}")); return required.describe(operation); } - format!("Prompt failed: {error:?}") + format!("{PROMPT_FAILED_PREFIX} {error:?}") } /// What an agent said on a `-32000` beyond the schema's stock text, if anything. @@ -9319,6 +9334,61 @@ mod tests { ); } + /// [`SessionSettled::prompt_failed`] is read off the transport monitor, + /// which the turn marks when the agent itself answers the prompt with an + /// error. The session runner classifies a failed run from that flag, so + /// it must be set by a real answered error and by nothing before it. + #[tokio::test(flavor = "current_thread")] + async fn an_answered_prompt_error_marks_the_prompt_failed() { + let provider_error = agent_client_protocol::Error::internal_error() + .data(serde_json::json!({ "errorKind": "server_error" })); + let agent = agent_client_protocol::Agent.builder().on_receive_request( + async move |_request: PromptRequest, responder, _cx| { + responder.respond_with_error(provider_error.clone()) + }, + agent_client_protocol::on_receive_request!(), + ); + let setup = AcpSessionSetup { + agent_session_id: "session-1".to_string(), + agent_capabilities: AgentCapabilities::default(), + task_tracking_mode: TaskTrackingMode::Raw, + auth_methods: vec![], + steering_supported: false, + }; + let handler = Arc::new( + AcpNotificationHandler::new( + Arc::new(BasicMessageWriter::new()), + false, + vec![], + CancellationToken::new(), + ) + .named("sess-1"), + ); + assert!(!handler.transport.answered_error()); + + let (_turn_control_tx, mut turn_control_rx) = mpsc::unbounded_channel(); + let outcome = agent_client_protocol::Client + .connect_with(agent, async |connection| { + Ok(run_prompt_turn( + &connection, + &setup, + &handler, + "local-session", + &CancellationToken::new(), + &mut turn_control_rx, + "hello", + &[], + ) + .await) + }) + .await + .expect("the connection itself stays healthy"); + + let error = outcome.expect_err("the agent's error should fail the prompt"); + assert!(error.starts_with("Prompt failed: "), "{error}"); + assert!(handler.transport.answered_error()); + } + /// A `-32000` with the bridge's own sentence on it, as pi-acp sends at /// `session/new`; the stock `auth_required` is what every other surveyed /// bridge sends. @@ -14047,6 +14117,7 @@ agent: http=false, sse=false). Select a provider that supports MCP over HTTP/SSE transport_loss: None, cancel_unacknowledged: false, background_work_left: false, + prompt_failed: false, }; // A hold that was cancelled or failed after the turn completed is the diff --git a/crates/acp-client/src/driver/recorded_turn.rs b/crates/acp-client/src/driver/recorded_turn.rs index b5969c9c5..e39db09e1 100644 --- a/crates/acp-client/src/driver/recorded_turn.rs +++ b/crates/acp-client/src/driver/recorded_turn.rs @@ -188,7 +188,7 @@ impl PromptAnswers { fn prompt_error_message(error: &serde_json::Value) -> String { match serde_json::from_value::(error.clone()) { Ok(error) => describe_prompt_error(error, &[]), - Err(_) => format!("Prompt failed: {error}"), + Err(_) => format!("{PROMPT_FAILED_PREFIX} {error}"), } }