From 222589ef2a16324380d5d492ea5cf1bbb7bfcdcb Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Tue, 17 Nov 2026 16:35:00 +1100 Subject: [PATCH 1/9] fix(staged): make branch setup the only path from starting to running A Blox workstation could end up on `main` with its branch marked `running`: `start_workspace` swallowed a failed `sq blox ws start` and a failed checkout, and the status pollers then promoted the row from what Blox reported about the workstation. Sessions started on the base branch. Requested: branch setup becomes the only code that moves a remote branch from `Starting` to `Running`, never reports success without the branch checked out, and is re-triggered from the backend whenever a workstation is running while its row is still `Starting`. How: - New `workspace_setup` module holds one setup routine, shared by the Tauri commands and the web server arms (`start_workspace`, `setup_remote_repo_clone`). Fresh and existing workstations both end in the idempotent `check_out_workspace_branch`. `Running` is written only after the checkout succeeded; a failure is returned and the row goes to `Error`. - A non-auth, non-onboarding failure or timeout of `sq blox ws start` no longer returns Ok. Setup waits a bounded time for the workstation to report running and then checks out, or fails with the start error. A workstation that is already booting is waited for, not started again. - `check_out_workspace_branch` asks `ls-remote` whether the branch is on origin, so a failed exec is an error and not "branch missing". - The single and batch status pollers moved to `workspace_setup::poll` and are shared with the web server. They no longer promote `Starting`; when the workstation is running and the row is still `Starting` they trigger setup. They also leave an `Error` row alone while the workstation runs, so a failed setup is not promoted by the next poll. Other transitions and the secondary-clone hold are unchanged. - A per-process guard joins concurrent callers for one branch onto one run and serialises runs per workstation, so the frontend call, a poller trigger, a second window or web client and the startup pass never run two checkouts in one clone. Joiners get the same result and write nothing. - A backend startup pass re-runs setup for branches a previous run left in `Starting`. `resume_workspace` triggers setup from the backend once the resume succeeded. - Frontend: the 5 minute starting timeout no longer fires while the controller's own setup call is pending or queued, and a resolved setup call marks the branch running without waiting for the next poll. Tests cover the swallowed-failure path ending in a checkout or a visible error, the pollers triggering setup instead of promoting, the guard joining concurrent callers, and the startup pass, all through a fake host and the `GitRunner` injection, with no `sq` call. Not in this commit: verifying HEAD at session start and hardening the `rev-parse` probe in `check_out_workspace_branch`. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- .../src/blox/tracked_exec_guard_tests.rs | 19 +- apps/staged/src-tauri/src/branches.rs | 740 ++---------- apps/staged/src-tauri/src/lib.rs | 6 + apps/staged/src-tauri/src/web_server.rs | 521 ++------- apps/staged/src-tauri/src/workspace_setup.rs | 543 +++++++++ .../src-tauri/src/workspace_setup/guard.rs | 134 +++ .../src-tauri/src/workspace_setup/poll.rs | 371 ++++++ .../src-tauri/src/workspace_setup/tests.rs | 1005 +++++++++++++++++ .../projects/workspaceLifecycle.svelte.ts | 23 +- .../projects/workspaceLifecycle.test.ts | 75 ++ 10 files changed, 2334 insertions(+), 1103 deletions(-) create mode 100644 apps/staged/src-tauri/src/workspace_setup.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/guard.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/poll.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests.rs diff --git a/apps/staged/src-tauri/src/blox/tracked_exec_guard_tests.rs b/apps/staged/src-tauri/src/blox/tracked_exec_guard_tests.rs index 49f8b1ab4..43488f9a9 100644 --- a/apps/staged/src-tauri/src/blox/tracked_exec_guard_tests.rs +++ b/apps/staged/src-tauri/src/blox/tracked_exec_guard_tests.rs @@ -18,17 +18,18 @@ use std::path::Path; const TRACKED: &[(&str, &str, usize)] = &[ // The user's own repo action: streams output and may run for minutes. ("actions/commands.rs", "run_branch_action_impl", 1), - // `git clone`, plus the runner behind `check_out_workspace_branch` (setup - // fetches, checkout, fast-forward merge, branch creation), for a repo - // added to a workspace. A killed checkout leaves `.git/index.lock`. - ("branches.rs", "clone_repo_into_workspace", 2), + // `git clone` of a repo a workstation doesn't have yet: a large one + // outlasts 60 s. + ("branches.rs", "clone_repo_into_workspace", 1), + // The runner behind `check_out_workspace_branch`, which every branch + // setup ends in (`workspace_setup`): the `ls-remote` probe and fetches + // (a killed one must not read as "not on origin"), checkout, + // fast-forward merge and branch creation. A killed checkout leaves + // `.git/index.lock`. + ("branches.rs", "check_out_branch_in_workspace", 1), // The diff-collection script reads both versions of every changed file // for every commit on the branch; on a large branch it outlasts 60 s. ("diff_cache.rs", "collect_branch_diff", 1), - // The setup fetch probing whether the branch exists on the remote (a - // killed probe would silently branch from base instead) and the setup - // checkout (a killed one leaves `.git/index.lock`). - ("branches.rs", "start_workspace", 2), // Pipeline steps: push runs pre-push hooks and a rebase can take minutes. ("session_runner.rs", "run_remote_pipeline_command", 1), // Project transfer scripts hash working trees, write bundles and clone; @@ -41,8 +42,6 @@ const TRACKED: &[(&str, &str, usize)] = &[ ("timeline.rs", "discard_remote_worktree_changes", 2), ("timeline.rs", "pull_branch_ff_only_impl", 1), ("timeline.rs", "reset_branch_to_remote_impl", 2), - // The `start_workspace` arm's copy of the setup fetch and checkout. - ("web_server.rs", "dispatch", 2), ]; #[test] diff --git a/apps/staged/src-tauri/src/branches.rs b/apps/staged/src-tauri/src/branches.rs index e856c3b6a..2a7f278c8 100644 --- a/apps/staged/src-tauri/src/branches.rs +++ b/apps/staged/src-tauri/src/branches.rs @@ -1,7 +1,6 @@ use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; -use std::time::Instant; use tauri::{AppHandle, Manager}; @@ -42,8 +41,8 @@ pub(crate) fn emit_setup_progress( pub(crate) const WORKSPACE_IDLE_TIMEOUT_MINUTES: u32 = 10080; // In-memory cache: workspace name → numeric workstation ID. -// Populated by `poll_workspace_status` and `start_workspace` when `blox ws info` -// returns an ID; read by `to_branch_with_workdir` when serializing for the frontend. +// Populated by the status pollers and branch setup when `blox ws info` or +// `ws list` returns an ID; read by `to_branch_with_workdir` when serializing for the frontend. pub(crate) fn workstation_id_cache() -> &'static Mutex> { static CACHE: OnceLock>> = OnceLock::new(); CACHE.get_or_init(|| Mutex::new(HashMap::new())) @@ -740,11 +739,11 @@ fn needs_clone(probe: Result) -> Result`), or `None` for +/// a branch with no project repo, whose git runs bare in the workstation's +/// default directory. +pub(crate) async fn check_out_branch_in_workspace( + ws_name: &str, + remote_dir: Option<&str>, + base_ref: &str, + branch_name: &str, +) -> Result<(), String> { let workspace = ws_name.to_string(); - let remote_dir = clone_dir.to_string(); + let remote_dir = remote_dir.map(str::to_string); let base = base_ref.to_string(); let branch = branch_name.to_string(); tauri::async_runtime::spawn_blocking(move || { // Tracked for everything but `rev-parse`: Blox SIGKILLs a transient - // git at 60 s, which a fetch on a large repo can outlast (reading as - // "no remote branch" and silently branching from base), and a + // git at 60 s, which a fetch on a large repo can outlast, and a // checkout or fast-forward far from where the tree is (or smudging LFS // files) can outlast it too, leaving `.git/index.lock` and a half-updated worktree. let git: GitRunner<'_> = Box::new(move |args: &[&str]| { @@ -799,14 +812,13 @@ pub(crate) async fn clone_repo_into_workspace( } else { ExecMode::Tracked }; - run_workspace_git_with_mode(&workspace, Some(&remote_dir), mode, args) + run_workspace_git_with_mode(&workspace, remote_dir.as_deref(), mode, args) .map_err(|e| e.to_string()) }); check_out_workspace_branch(&git, &base, &branch) }) .await .map_err(|e| format!("Branch checkout task failed: {e}"))? - .map_err(|e| format!("Failed to set up '{repo_slug}' in workspace '{ws_name}': {e}")) } /// Fetch `base_ref` into the clone `git` runs in, and put the clone on @@ -821,7 +833,7 @@ pub(crate) async fn clone_repo_into_workspace( /// behind picks up what was pushed, and one that has diverged keeps its local /// commits. Only a branch the clone doesn't have yet is created, from origin's /// copy or from the base. -fn check_out_workspace_branch( +pub(crate) fn check_out_workspace_branch( git: &GitRunner<'_>, base_ref: &str, branch_name: &str, @@ -833,11 +845,23 @@ fn check_out_workspace_branch( // fetch it so we pick up its commits: a new branch starts from there and // an existing one fast-forwards to it. Otherwise a new branch falls back // to starting from the base branch. - let has_remote_branch = - branch_name != base_ref && git(&["fetch", "origin", branch_name]).is_ok(); + // + // Whether it exists is read from `ls-remote`'s output, not from a fetch + // failing: a fetch that never reached origin (a dropped exec, a network + // blip) would otherwise read as "not on origin" and branch from the base, + // leaving the PR's commits behind. Any failure here is a failure. + let local_ref = format!("refs/heads/{branch_name}"); + let has_remote_branch = branch_name != base_ref + && !git(&["ls-remote", "--heads", "origin", &local_ref]) + .map_err(|e| format!("could not check whether '{branch_name}' is on origin: {e}"))? + .trim() + .is_empty(); + if has_remote_branch { + git(&["fetch", "origin", branch_name]) + .map_err(|e| format!("could not fetch branch '{branch_name}': {e}"))?; + } let remote_branch = format!("origin/{branch_name}"); - let local_ref = format!("refs/heads/{branch_name}"); if git(&["rev-parse", "--verify", "--quiet", &local_ref]).is_ok() { git(&["checkout", branch_name]) .map_err(|e| format!("could not check out branch '{branch_name}': {e}"))?; @@ -894,55 +918,24 @@ pub(crate) fn workspace_status_written( } } -/// Clone a repo into an already-running remote workspace for a given branch. +/// Clone a repo into an already-running remote workspace for a given branch, +/// check its branch out and mark it `Running`. /// -/// Resolves the workspace name, repo slug, subpath, and base ref from the -/// branch and project, then delegates to `clone_repo_into_workspace`. Updates -/// the branch's workspace status to Running on success. +/// The shared branch setup ([`crate::workspace_setup`]) under its guard, so +/// the frontend's `start_workspace` for the same `Starting` branch joins this +/// run instead of cloning beside it. pub(crate) async fn setup_remote_repo_clone( store: &Arc, app_handle: &tauri::AppHandle, branch_id: &str, ) -> Result<(), String> { - let branch = store - .get_branch(branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - - let project = store - .get_project(&branch.project_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; - - let ws_name = branch - .workspace_name - .as_deref() - .ok_or("Branch has no workspace name")?; - let clone_dir = - resolve_branch_clone_dir(store, &branch)?.ok_or("Branch has no project repo to clone")?; - let ref_name = normalize_branch_ref(&branch.base_branch); - let repo_slug = resolve_branch_repo_slug(store, &project, &branch)?; - - clone_repo_into_workspace( - ws_name, - &clone_dir, - &repo_slug, - &ref_name, - &branch.branch_name, - ) - .await?; - - let changed = store - .update_branch_workspace_status(branch_id, &store::WorkspaceStatus::Running) - .map_err(|e| e.to_string())?; - workspace_status_written( + crate::workspace_setup::ensure_branch_set_up( app_handle, + store, branch_id, - changed, - &store::WorkspaceStatus::Running, - ); - - Ok(()) + crate::workspace_setup::SetupTrigger::Requested, + ) + .await } /// Result of searching for an existing worktree for a branch name. @@ -2416,11 +2409,16 @@ pub(crate) async fn create_remote_branch_impl( .map_err(|e| e.to_string())? } -/// Start the Blox workspace for a remote branch. +/// Start the Blox workspace for a remote branch and set its branch up. /// /// Separated from `create_remote_branch` so the frontend can dismiss the /// dialog immediately and show the card in its "Provisioning…" state while /// this runs in the background. +/// +/// Returns once the branch is checked out on the workstation and marked +/// `Running`, or with the error that left it in `Error`: the shared routine +/// in [`crate::workspace_setup`], which a concurrent call for the same +/// branch joins. #[tauri::command(rename_all = "camelCase")] pub async fn start_workspace( store: tauri::State<'_, Mutex>>>, @@ -2428,207 +2426,13 @@ pub async fn start_workspace( branch_id: String, ) -> Result<(), String> { let store = get_store(&store)?; - crate::project_transfer::ensure_branch_not_transferring(&branch_id)?; - - let branch = store - .get_branch(&branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - - let project = store - .get_project(&branch.project_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; - - let ws_name = branch - .workspace_name - .as_deref() - .ok_or("Branch has no workspace name")?; - let repo_subpath = resolve_branch_workspace_subpath(&store, &branch)?; - let clone_dir = resolve_branch_clone_dir(&store, &branch)?; - let ref_name = normalize_branch_ref(&branch.base_branch); - let repo_slug = resolve_branch_repo_slug(&store, &project, &branch)?; - - // Pre-flight: verify the user is authenticated with Blox before starting - // a workspace. Without this check, `blox ws start` can hang or fail - // opaquely, leaving the frontend spinning for minutes (issue #99). - if let Err(e) = run_blox_blocking(blox::check_auth).await { - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - return Err(e.to_string()); - } - - // Secondary repo setup in an already-running shared workspace — or a - // branch re-entering `Starting` (e.g. after `resume_workspace`) on a - // workstation that already has this clone. - if let Some(clone_dir) = clone_dir.as_deref() { - if let Ok(info) = run_blox_blocking({ - let ws_name = ws_name.to_string(); - move || blox::ws_info(&ws_name) - }) - .await - { - let ws_status = info - .status - .as_deref() - .map(|s| s.to_ascii_lowercase()) - .unwrap_or_default(); - if let Some(ws_id) = info.workstation_id { - if let Ok(mut cache) = workstation_id_cache().lock() { - cache.insert(ws_name.to_string(), ws_id); - } - } - if ws_status == "running" { - clone_repo_into_workspace( - ws_name, - clone_dir, - &repo_slug, - &ref_name, - &branch.branch_name, - ) - .await?; - let changed = store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Running) - .map_err(|e| e.to_string())?; - workspace_status_written( - &app_handle, - &branch_id, - changed, - &store::WorkspaceStatus::Running, - ); - - return Ok(()); - } - } - } - - // Construct the HTTPS git URL directly from github_repo. - let resolved_source = Some(format!( - "https://github.com/{}.git?ref={}", - repo_slug, ref_name - )); - let start_command_preview = match resolved_source.as_deref() { - Some(source) => format!( - "sq blox ws start {} --idle-timeout {} {}", - ws_name, WORKSPACE_IDLE_TIMEOUT_MINUTES, source - ), - None => format!( - "sq blox ws start {} --idle-timeout {}", - ws_name, WORKSPACE_IDLE_TIMEOUT_MINUTES - ), - }; - log::info!( - "[start_workspace] branch={} workspace={} invoking command=\"{}\"", - branch_id, - ws_name, - start_command_preview - ); - let ws_start_started_at = Instant::now(); - - match run_blox_blocking({ - let ws_name = ws_name.to_string(); - let source = resolved_source.clone(); - move || { - blox::ws_start( - &ws_name, - source.as_deref(), - Some(WORKSPACE_IDLE_TIMEOUT_MINUTES), - ) - } - }) + crate::workspace_setup::ensure_branch_set_up( + &app_handle, + &store, + &branch_id, + crate::workspace_setup::SetupTrigger::Requested, + ) .await - { - Ok(_) => { - log::info!( - "[start_workspace] branch={} workspace={} ws_start completed elapsed_ms={}", - branch_id, - ws_name, - ws_start_started_at.elapsed().as_millis() - ); - // If the branch already exists on the remote (e.g. from an - // existing PR), fetch it and start from there so we pick up its - // commits. Otherwise create a fresh branch from the base ref. - // Tracked: a transient fetch killed at 60 s would read as "no - // remote branch" and silently branch from base. - let has_remote_branch = run_workspace_git_async( - ws_name, - repo_subpath.as_deref(), - ExecMode::Tracked, - &["fetch", "origin", &branch.branch_name], - ) - .await - .is_ok(); - - let remote_ref = format!("origin/{}", branch.branch_name); - let checkout_args: &[&str] = if has_remote_branch { - &["checkout", "-B", &branch.branch_name, &remote_ref] - } else { - &["checkout", "-b", &branch.branch_name] - }; - // Tracked: Blox SIGKILLs a transient git at 60 s, and a killed - // checkout leaves `.git/index.lock` and a half-updated worktree. - let checkout_result = run_workspace_git_async( - ws_name, - repo_subpath.as_deref(), - ExecMode::Tracked, - checkout_args, - ) - .await; - - if let Err(e) = checkout_result { - log::warn!( - "failed to create branch '{}' in workspace '{}': {e}", - branch.branch_name, - ws_name - ); - } - - Ok(()) - } - Err(blox::BloxError::NotAuthenticated) => { - log::warn!( - "[start_workspace] branch={} workspace={} ws_start failed elapsed_ms={} error=NotAuthenticated", - branch_id, - ws_name, - ws_start_started_at.elapsed().as_millis() - ); - // Auth errors are definitive — mark as Error so the frontend - // stops polling and shows an actionable message. - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - Err("Not authenticated with Blox. Run: sq login".to_string()) - } - Err(e) => { - log::warn!( - "[start_workspace] branch={} workspace={} ws_start failed elapsed_ms={} error={}", - branch_id, - ws_name, - ws_start_started_at.elapsed().as_millis(), - e - ); - if is_blox_onboarding_precondition_error(&e) { - // Blox onboarding precondition failures are definitive. - // Keep the exact CLI error text so the user sees the - // onboarding URL and required action. - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - return Err(e.to_string()); - } - // Don't set Error status here — `blox ws start` can fail (e.g. - // timeout, transient network issue) even though the workspace was - // created and is still booting. Let the frontend's status polling - // determine the real state; it will keep polling while the DB says - // Starting and will eventually converge on the correct status. - log::warn!( - "blox ws start failed for '{}', leaving status as Starting for polling to resolve: {e}", - ws_name - ); - Ok(()) - } - } } /// Return the value of the `BLOX_ENV` environment variable, if set. @@ -2659,6 +2463,15 @@ pub async fn get_workspace_info( .map_err(|e| e.to_string()) } +/// Whether a raw Blox status string says the workstation exists and is +/// coming up: being created and bootstrapped, or resumed. +pub(crate) fn is_blox_booting_status(blox_status: &str) -> bool { + matches!( + blox_status.to_ascii_lowercase().as_str(), + "starting" | "provisioning" | "creating" + ) +} + /// Map a raw Blox status string to our `WorkspaceStatus` enum. /// /// During initial startup, Blox may briefly report "stopped" before the @@ -2678,9 +2491,7 @@ pub(crate) fn map_blox_status_to_workspace_status( store::WorkspaceStatus::Stopped } } - Some("starting") | Some("provisioning") | Some("creating") => { - store::WorkspaceStatus::Starting - } + Some(status) if is_blox_booting_status(status) => store::WorkspaceStatus::Starting, Some("suspended") => store::WorkspaceStatus::Suspended, Some("deleted") => store::WorkspaceStatus::Stopped, Some("shutting_down") => { @@ -2698,10 +2509,11 @@ pub(crate) fn map_blox_status_to_workspace_status( /// Poll a remote branch's workspace status, update the DB, and return the new status. /// -/// This is the primary mechanism for the frontend to detect when a workspace -/// transitions from `Starting` to `Running` (or `Error`). It queries the -/// Blox CLI, maps the reported status to our `WorkspaceStatus` enum, persists -/// the change, and returns the updated status string. +/// Queries the Blox CLI, maps the reported status to our `WorkspaceStatus` +/// enum, persists the change, and returns the updated status string. It +/// never moves a branch from `Starting` to `Running`: a workstation that is +/// up while its row is still `Starting` gets branch setup triggered instead +/// (see [`crate::workspace_setup::poll`]). #[tauri::command(rename_all = "camelCase")] pub async fn poll_workspace_status( store: tauri::State<'_, Mutex>>>, @@ -2709,171 +2521,7 @@ pub async fn poll_workspace_status( branch_id: String, ) -> Result { let store = get_store(&store)?; - - let branch = store - .get_branch(&branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - - let ws_name = branch - .workspace_name - .as_deref() - .ok_or("Branch is not a remote workspace branch")?; - - // Secondary repo setup uses `workspace_status=Starting` as a per-branch - // loading state while clone/fetch/checkout runs inside an already-running - // shared workspace. Keep it in Starting until that setup command marks - // this branch as Running. - // - // Important: this must apply only to "secondary clone" branches. Initial - // remote branches also have a workspace subpath, so keying only on - // `resolve_branch_workspace_subpath().is_some()` would pin them in - // Starting forever and block normal status polling. - let is_secondary_clone_setup = if branch.workspace_status - == Some(store::WorkspaceStatus::Starting) - && resolve_branch_workspace_subpath(&store, &branch)?.is_some() - { - if let Some(ws_name) = branch.workspace_name.as_deref() { - let peers = store - .list_branches_for_project(&branch.project_id) - .map_err(|e| e.to_string())?; - peers.into_iter().any(|peer| { - peer.id != branch.id - && peer.branch_type == store::BranchType::Remote - && peer.workspace_name.as_deref() == Some(ws_name) - && peer.workspace_status == Some(store::WorkspaceStatus::Running) - }) - } else { - false - } - } else { - false - }; - - if is_secondary_clone_setup { - log::debug!( - "[poll_workspace_status] branch={} ws={} held at Starting for secondary clone setup", - branch_id, - ws_name - ); - return Ok(PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }); - } - - let info = match run_blox_blocking({ - let ws_name = ws_name.to_string(); - move || blox::ws_info(&ws_name) - }) - .await - { - Ok(info) => info, - Err(blox::BloxError::NotAuthenticated) => { - // Auth errors are definitive — stop polling and surface the error. - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - return Err("Not authenticated with Blox. Run: sq login".to_string()); - } - Err(e) => { - let is_not_found = matches!(&e, blox::BloxError::CommandFailed(msg) if msg.to_lowercase().contains("not found")); - - // During initial creation, `blox ws start` may still be running - // when the frontend's first poll fires `blox ws info`. The - // workspace doesn't exist yet, so the CLI returns "not found". - // If the DB still says Starting, swallow the error and tell the - // frontend to keep polling. - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) { - log::debug!( - "blox ws info failed for '{}' while still Starting, treating as Starting: {e}", - ws_name - ); - return Ok(PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }); - } - // If the workspace was Running but Blox explicitly says "not - // found" (e.g. deleted externally), transition to Stopped. - // Other errors (network blips, timeouts, etc.) are transient — - // keep the Running status so the poller retries. - if branch.workspace_status == Some(store::WorkspaceStatus::Running) { - if is_not_found { - log::debug!( - "blox ws info returned not-found for '{}' while Running, treating as Stopped: {e}", - ws_name - ); - store - .update_branch_workspace_status( - &branch_id, - &store::WorkspaceStatus::Stopped, - ) - .ok(); - return Ok(PollWorkspaceResult { - status: store::WorkspaceStatus::Stopped.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }); - } - log::warn!( - "blox ws info failed for '{}' while Running, keeping Running status: {e}", - ws_name - ); - return Ok(PollWorkspaceResult { - status: store::WorkspaceStatus::Running.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }); - } - return Err(e.to_string()); - } - }; - - log::debug!( - "[poll_workspace_status] branch={} ws={} db_status={:?} blox_status={:?}", - branch_id, - ws_name, - branch.workspace_status, - info.status - ); - - let new_status = map_blox_status_to_workspace_status( - info.status.as_deref(), - branch.workspace_status.as_ref(), - ); - - let previous = branch - .workspace_status - .as_ref() - .map(|s| s.as_str()) - .unwrap_or("none"); - if previous != new_status.as_str() { - log::debug!( - "[poll_workspace_status] branch={} ws={} transition {} -> {} (raw={:?})", - branch_id, - ws_name, - previous, - new_status.as_str(), - info.status - ); - } - // Cache the numeric workstation ID for proxy URL construction. - if let Some(ws_id) = info.workstation_id { - if let Some(name) = branch.workspace_name.as_deref() { - if let Ok(mut cache) = workstation_id_cache().lock() { - cache.insert(name.to_string(), ws_id); - } - } - } - - let changed = store - .update_branch_workspace_status(&branch_id, &new_status) - .map_err(|e| e.to_string())?; - workspace_status_written(&app_handle, &branch_id, changed, &new_status); - - Ok(PollWorkspaceResult { - status: new_status.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }) + crate::workspace_setup::poll::poll_workspace_status(&store, &app_handle, &branch_id).await } /// Map a blox orchestrator `CommandType` to a stable string identifier that @@ -2942,7 +2590,9 @@ pub(crate) fn emit_workspace_setup_progress( /// /// Returns a map from branch ID to `PollWorkspaceResult`. Branches that cannot be /// resolved (e.g. missing workspace name, not found in store) are silently omitted -/// from the result rather than failing the entire batch. +/// from the result rather than failing the entire batch. Like +/// [`poll_workspace_status`], it triggers branch setup rather than promoting +/// a `Starting` branch. #[tauri::command(rename_all = "camelCase")] pub async fn poll_all_workspace_statuses( app_handle: AppHandle, @@ -2950,202 +2600,8 @@ pub async fn poll_all_workspace_statuses( branch_ids: Vec, ) -> Result, String> { let store = get_store(&store)?; - - // Fetch all workspaces in one CLI call. - let entries = run_blox_blocking(blox::ws_list).await.map_err(|e| { - if matches!(e, blox::BloxError::NotAuthenticated) { - "Not authenticated with Blox. Run: sq login".to_string() - } else { - e.to_string() - } - })?; - - // Build a lookup from workspace name → list entry. - let ws_map: HashMap = - entries.iter().map(|e| (e.name.clone(), e)).collect(); - - // Map workspace name → branch IDs for workspaces in "starting" state so we - // can fetch their bootstrap command progress and emit events. - let mut starting_workspaces: HashMap> = HashMap::new(); - - let mut results = HashMap::new(); - - for branch_id in &branch_ids { - let branch = match store.get_branch(branch_id) { - Ok(Some(b)) => b, - _ => continue, - }; - - let ws_name = match branch.workspace_name.as_deref() { - Some(n) => n, - None => continue, - }; - - // Secondary clone setup: if this branch is Starting and shares a workspace - // with a Running peer (multi-repo setup), hold it at Starting until the - // setup command marks it as Running. This mirrors the individual - // poll_workspace_status logic. - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) - && resolve_branch_workspace_subpath(&store, &branch) - .unwrap_or_else(|e| { - log::warn!( - "Failed to resolve workspace subpath for branch {}: {e}", - branch_id - ); - None - }) - .is_some() - { - let is_secondary = if let Some(ws) = branch.workspace_name.as_deref() { - store - .list_branches_for_project(&branch.project_id) - .unwrap_or_default() - .into_iter() - .any(|peer| { - peer.id != *branch_id - && peer.branch_type == store::BranchType::Remote - && peer.workspace_name.as_deref() == Some(ws) - && peer.workspace_status == Some(store::WorkspaceStatus::Running) - }) - } else { - false - }; - - if is_secondary { - log::debug!( - "[poll_all_workspace_statuses] branch={} ws={} held at Starting for secondary clone setup", - branch_id, - ws_name - ); - results.insert( - branch_id.clone(), - PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }, - ); - continue; - } - } - - // Look up the workspace in the list. - match ws_map.get(ws_name) { - Some(entry) => { - let new_status = map_blox_status_to_workspace_status( - entry.status.as_deref(), - branch.workspace_status.as_ref(), - ); - - let previous = branch - .workspace_status - .as_ref() - .map(|s| s.as_str()) - .unwrap_or("none"); - if previous != new_status.as_str() { - log::debug!( - "[poll_all_workspace_statuses] branch={} ws={} transition {} -> {} (raw={:?})", - branch_id, - ws_name, - previous, - new_status.as_str(), - entry.status - ); - } - - // Cache the numeric workstation ID. - if let Some(ws_id) = entry.workstation_id { - if let Ok(mut cache) = workstation_id_cache().lock() { - cache.insert(ws_name.to_string(), ws_id); - } - } - - if let Ok(changed) = store.update_branch_workspace_status(branch_id, &new_status) { - workspace_status_written(&app_handle, branch_id, changed, &new_status); - } - - // Track workspaces still starting so we can fetch bootstrap progress. - if new_status == store::WorkspaceStatus::Starting { - starting_workspaces - .entry(ws_name.to_string()) - .or_default() - .push(branch_id.clone()); - } - - results.insert( - branch_id.clone(), - PollWorkspaceResult { - status: new_status.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }, - ); - } - None => { - // Workspace not in list — same "not found" logic as individual polling. - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) { - log::debug!( - "[poll_all_workspace_statuses] branch={} ws={} not in list while Starting, keeping Starting", - branch_id, - ws_name - ); - starting_workspaces - .entry(ws_name.to_string()) - .or_default() - .push(branch_id.clone()); - results.insert( - branch_id.clone(), - PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }, - ); - } else if branch.workspace_status == Some(store::WorkspaceStatus::Running) { - log::debug!( - "[poll_all_workspace_statuses] branch={} ws={} not in list while Running, treating as Stopped", - branch_id, - ws_name - ); - store - .update_branch_workspace_status(branch_id, &store::WorkspaceStatus::Stopped) - .ok(); - results.insert( - branch_id.clone(), - PollWorkspaceResult { - status: store::WorkspaceStatus::Stopped.as_str().to_string(), - workstation_id: cached_workstation_id(ws_name), - }, - ); - } - // Other statuses (Stopped, Error, etc.) — omit from results, no change needed. - } - } - } - - // For workspaces that are still starting, fetch bootstrap command progress - // and emit events so the UI can show which step is running. - if !starting_workspaces.is_empty() { - let app_handle_clone = app_handle.clone(); - - // Fetch commands for each starting workspace in a blocking task to - // avoid holding up the response. - tauri::async_runtime::spawn_blocking(move || { - for (ws_name, branch_ids) in &starting_workspaces { - match blox::ws_commands(ws_name) { - Ok(cmds) => { - emit_workspace_setup_progress(&app_handle_clone, branch_ids, &cmds); - } - Err(e) => { - log::debug!( - "[poll_all_workspace_statuses] ws_commands({}) failed: {}", - ws_name, - e - ); - } - } - } - }); - } - - Ok(results) + crate::workspace_setup::poll::poll_all_workspace_statuses(&store, &app_handle, &branch_ids) + .await } /// Resume a suspended Blox workspace. @@ -3155,15 +2611,30 @@ pub async fn poll_all_workspace_statuses( #[tauri::command(rename_all = "camelCase")] pub async fn resume_workspace( store: tauri::State<'_, Mutex>>>, + app_handle: AppHandle, workspace_name: String, ) -> Result, String> { let store = get_store(&store)?; - crate::project_transfer::ensure_workspace_not_transferring(&workspace_name)?; + resume_workspace_impl(&store, &app_handle, &workspace_name).await +} + +/// [`resume_workspace`], shared with the web server arm. +/// +/// The branches stay `Starting` when this returns: resuming brings the +/// workstation back, and only branch setup marks a branch `Running`, once +/// its checkout has succeeded. Setup is triggered here for each branch so a +/// resume finishes without depending on a window to ask for it. +pub(crate) async fn resume_workspace_impl( + store: &Arc, + app_handle: &AppHandle, + workspace_name: &str, +) -> Result, String> { + crate::project_transfer::ensure_workspace_not_transferring(workspace_name)?; // Transition all peer branches to Starting so the UI begins polling. let branch_ids = store .update_workspace_status_by_workspace_name( - &workspace_name, + workspace_name, &store::WorkspaceStatus::Starting, ) .map_err(|e| e.to_string())?; @@ -3172,8 +2643,7 @@ pub async fn resume_workspace( return Err(format!("No branches found for workspace: {workspace_name}")); } - // Call resume in the background (it may take a while). - let ws = workspace_name.clone(); + let ws = workspace_name.to_string(); if let Err(e) = run_blox_blocking(move || blox::ws_resume(&ws)).await { log::warn!( "[resume_workspace] workspace={} resume failed: {}", @@ -3182,13 +2652,17 @@ pub async fn resume_workspace( ); store .update_workspace_status_by_workspace_name( - &workspace_name, + workspace_name, &store::WorkspaceStatus::Error, ) .ok(); return Err(e.to_string()); } + for branch_id in &branch_ids { + crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id); + } + Ok(branch_ids) } diff --git a/apps/staged/src-tauri/src/lib.rs b/apps/staged/src-tauri/src/lib.rs index eadb9be86..9692936f5 100644 --- a/apps/staged/src-tauri/src/lib.rs +++ b/apps/staged/src-tauri/src/lib.rs @@ -66,6 +66,7 @@ mod upstream_repair; pub mod util_commands; pub mod web_server; pub mod window_commands; +pub(crate) mod workspace_setup; #[cfg(test)] mod board_load_bench; @@ -2280,6 +2281,11 @@ pub fn run() { store, session_queue::KickSource::Startup, ); + // Remote branches a previous run left in `Starting`: the + // workstation may be up with nothing checked out in it. + // After the transfer recovery, whose branches are its own + // to provision. + workspace_setup::resume_pending_setups(&transfer_app, store); } project_transfer::prune_backups(transfer_store.as_deref()); }); diff --git a/apps/staged/src-tauri/src/web_server.rs b/apps/staged/src-tauri/src/web_server.rs index 885b06368..a62c90d6a 100644 --- a/apps/staged/src-tauri/src/web_server.rs +++ b/apps/staged/src-tauri/src/web_server.rs @@ -2151,268 +2151,20 @@ pub(crate) async fn dispatch( "poll_workspace_status" => { let store = get_store(store_mutex)?; let branch_id: String = arg(&args, "branchId")?; - let branch = store - .get_branch(&branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - let ws_name = branch - .workspace_name - .as_deref() - .ok_or("Branch is not a remote workspace branch")?; - - // Secondary clone setup: hold at Starting until setup marks it Running. - let is_secondary_clone_setup = if branch.workspace_status - == Some(store::WorkspaceStatus::Starting) - && crate::branches::resolve_branch_workspace_subpath(&store, &branch)?.is_some() - { - if let Some(ws_name) = branch.workspace_name.as_deref() { - let peers = store - .list_branches_for_project(&branch.project_id) - .map_err(|e| e.to_string())?; - peers.into_iter().any(|peer| { - peer.id != branch.id - && peer.branch_type == store::BranchType::Remote - && peer.workspace_name.as_deref() == Some(ws_name) - && peer.workspace_status == Some(store::WorkspaceStatus::Running) - }) - } else { - false - } - } else { - false - }; - - if is_secondary_clone_setup { - return Ok(serde_json::to_value(crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }) - .unwrap()); - } - - let info = match crate::branches::run_blox_blocking({ - let ws_name = ws_name.to_string(); - move || crate::blox::ws_info(&ws_name) - }) - .await - { - Ok(info) => info, - Err(crate::blox::BloxError::NotAuthenticated) => { - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - return Err("Not authenticated with Blox. Run: sq login".to_string()); - } - Err(e) => { - let is_not_found = matches!(&e, crate::blox::BloxError::CommandFailed(msg) if msg.to_lowercase().contains("not found")); - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) { - return Ok(serde_json::to_value(crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }) - .unwrap()); - } - if branch.workspace_status == Some(store::WorkspaceStatus::Running) { - if is_not_found { - store - .update_branch_workspace_status( - &branch_id, - &store::WorkspaceStatus::Stopped, - ) - .ok(); - return Ok(serde_json::to_value(crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Stopped.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }) - .unwrap()); - } - return Ok(serde_json::to_value(crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Running.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }) - .unwrap()); - } - return Err(e.to_string()); - } - }; - - let new_status = crate::branches::map_blox_status_to_workspace_status( - info.status.as_deref(), - branch.workspace_status.as_ref(), - ); - if let Some(ws_id) = info.workstation_id { - if let Some(name) = branch.workspace_name.as_deref() { - if let Ok(mut cache) = crate::branches::workstation_id_cache().lock() { - cache.insert(name.to_string(), ws_id); - } - } - } - let changed = store - .update_branch_workspace_status(&branch_id, &new_status) - .map_err(|e| e.to_string())?; - crate::branches::workspace_status_written(app_handle, &branch_id, changed, &new_status); - Ok(serde_json::to_value(crate::PollWorkspaceResult { - status: new_status.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }) - .unwrap()) + let result = + crate::workspace_setup::poll::poll_workspace_status(&store, app_handle, &branch_id) + .await?; + Ok(serde_json::to_value(result).unwrap()) } "poll_all_workspace_statuses" => { let store = get_store(store_mutex)?; let branch_ids: Vec = arg(&args, "branchIds")?; - - let entries = crate::branches::run_blox_blocking(crate::blox::ws_list) - .await - .map_err(|e| { - if matches!(e, crate::blox::BloxError::NotAuthenticated) { - "Not authenticated with Blox. Run: sq login".to_string() - } else { - e.to_string() - } - })?; - - let ws_map: std::collections::HashMap = - entries.iter().map(|e| (e.name.clone(), e)).collect(); - - let mut starting_workspaces: std::collections::HashMap> = - std::collections::HashMap::new(); - let mut results = std::collections::HashMap::new(); - - for branch_id in &branch_ids { - let branch = match store.get_branch(branch_id) { - Ok(Some(b)) => b, - _ => continue, - }; - let ws_name = match branch.workspace_name.as_deref() { - Some(n) => n, - None => continue, - }; - - // Secondary clone setup check - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) - && crate::branches::resolve_branch_workspace_subpath(&store, &branch) - .unwrap_or(None) - .is_some() - { - let is_secondary = if let Some(ws) = branch.workspace_name.as_deref() { - store - .list_branches_for_project(&branch.project_id) - .unwrap_or_default() - .into_iter() - .any(|peer| { - peer.id != *branch_id - && peer.branch_type == store::BranchType::Remote - && peer.workspace_name.as_deref() == Some(ws) - && peer.workspace_status - == Some(store::WorkspaceStatus::Running) - }) - } else { - false - }; - if is_secondary { - results.insert( - branch_id.clone(), - crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }, - ); - continue; - } - } - - match ws_map.get(ws_name) { - Some(entry) => { - let new_status = crate::branches::map_blox_status_to_workspace_status( - entry.status.as_deref(), - branch.workspace_status.as_ref(), - ); - if let Some(ws_id) = entry.workstation_id { - if let Ok(mut cache) = crate::branches::workstation_id_cache().lock() { - cache.insert(ws_name.to_string(), ws_id); - } - } - if let Ok(changed) = - store.update_branch_workspace_status(branch_id, &new_status) - { - crate::branches::workspace_status_written( - app_handle, - branch_id, - changed, - &new_status, - ); - } - if new_status == store::WorkspaceStatus::Starting { - starting_workspaces - .entry(ws_name.to_string()) - .or_default() - .push(branch_id.clone()); - } - results.insert( - branch_id.clone(), - crate::PollWorkspaceResult { - status: new_status.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }, - ); - } - None => { - if branch.workspace_status == Some(store::WorkspaceStatus::Starting) { - starting_workspaces - .entry(ws_name.to_string()) - .or_default() - .push(branch_id.clone()); - results.insert( - branch_id.clone(), - crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Starting.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }, - ); - } else if branch.workspace_status == Some(store::WorkspaceStatus::Running) { - store - .update_branch_workspace_status( - branch_id, - &store::WorkspaceStatus::Stopped, - ) - .ok(); - results.insert( - branch_id.clone(), - crate::PollWorkspaceResult { - status: store::WorkspaceStatus::Stopped.as_str().to_string(), - workstation_id: crate::branches::cached_workstation_id(ws_name), - }, - ); - } - } - } - } - - // Fetch bootstrap progress for starting workspaces in background. - if !starting_workspaces.is_empty() { - let app_handle_clone = app_handle.clone(); - tokio::task::spawn_blocking(move || { - for (ws_name, branch_ids) in &starting_workspaces { - match crate::blox::ws_commands(ws_name) { - Ok(cmds) => { - crate::branches::emit_workspace_setup_progress( - &app_handle_clone, - branch_ids, - &cmds, - ); - } - Err(e) => { - log::debug!( - "[poll_all_workspace_statuses] ws_commands({}) failed: {}", - ws_name, - e - ); - } - } - } - }); - } - + let results = crate::workspace_setup::poll::poll_all_workspace_statuses( + &store, + app_handle, + &branch_ids, + ) + .await?; Ok(serde_json::to_value(results).unwrap()) } "setup_worktree" => { @@ -2492,171 +2244,16 @@ pub(crate) async fn dispatch( "start_workspace" => { let store = get_store(store_mutex)?; let branch_id: String = arg(&args, "branchId")?; - crate::project_transfer::ensure_branch_not_transferring(&branch_id)?; let app_handle = app_handle.clone(); detached(async move { - let branch = store - .get_branch(&branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - - let project = store - .get_project(&branch.project_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; - - let ws_name = branch - .workspace_name - .as_deref() - .ok_or("Branch has no workspace name")?; - let repo_subpath = crate::branches::resolve_branch_workspace_subpath(&store, &branch)?; - let clone_dir = crate::branches::resolve_branch_clone_dir(&store, &branch)?; - let ref_name = crate::branches::normalize_branch_ref(&branch.base_branch); - let repo_slug = crate::branches::resolve_branch_repo_slug(&store, &project, &branch)?; - - // Pre-flight auth check - if let Err(e) = crate::branches::run_blox_blocking(crate::blox::check_auth).await { - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - return Err(e.to_string()); - } - - // Secondary repo setup in an already-running shared workspace — - // or a branch re-entering `Starting` on a workstation that - // already has this clone. - if let Some(clone_dir) = clone_dir.as_deref() { - if let Ok(info) = crate::branches::run_blox_blocking({ - let ws_name = ws_name.to_string(); - move || crate::blox::ws_info(&ws_name) - }) - .await - { - let ws_status = info - .status - .as_deref() - .map(|s| s.to_ascii_lowercase()) - .unwrap_or_default(); - if let Some(ws_id) = info.workstation_id { - if let Ok(mut cache) = crate::branches::workstation_id_cache().lock() { - cache.insert(ws_name.to_string(), ws_id); - } - } - if ws_status == "running" { - crate::branches::clone_repo_into_workspace( - ws_name, - clone_dir, - &repo_slug, - &ref_name, - &branch.branch_name, - ) - .await?; - let changed = store - .update_branch_workspace_status( - &branch_id, - &store::WorkspaceStatus::Running, - ) - .map_err(|e| e.to_string())?; - crate::branches::workspace_status_written( - &app_handle, - &branch_id, - changed, - &store::WorkspaceStatus::Running, - ); - return Ok(Value::Null); - } - } - } - - let resolved_source = Some(format!( - "https://github.com/{}.git?ref={}", - repo_slug, ref_name - )); - log::info!( - "[start_workspace] branch={} workspace={} starting", - branch_id, - ws_name - ); - let ws_start_started_at = std::time::Instant::now(); - - match crate::branches::run_blox_blocking({ - let ws_name = ws_name.to_string(); - let source = resolved_source.clone(); - move || { - crate::blox::ws_start( - &ws_name, - source.as_deref(), - Some(crate::branches::WORKSPACE_IDLE_TIMEOUT_MINUTES), - ) - } - }) - .await - { - Ok(_) => { - log::info!( - "[start_workspace] branch={} workspace={} ws_start completed elapsed_ms={}", - branch_id, - ws_name, - ws_start_started_at.elapsed().as_millis() - ); - // Tracked: a transient fetch killed at 60 s would read - // as "no remote branch" and silently branch from base. - let has_remote_branch = crate::branches::run_workspace_git_async( - ws_name, - repo_subpath.as_deref(), - crate::blox::ExecMode::Tracked, - &["fetch", "origin", &branch.branch_name], - ) - .await - .is_ok(); - - let remote_ref = format!("origin/{}", branch.branch_name); - let checkout_args: &[&str] = if has_remote_branch { - &["checkout", "-B", &branch.branch_name, &remote_ref] - } else { - &["checkout", "-b", &branch.branch_name] - }; - // Tracked: a checkout SIGKILLed at the transient 60 s - // limit leaves `.git/index.lock` behind. - let checkout_result = crate::branches::run_workspace_git_async( - ws_name, - repo_subpath.as_deref(), - crate::blox::ExecMode::Tracked, - checkout_args, - ) - .await; - if let Err(e) = checkout_result { - log::warn!( - "failed to create branch '{}' in workspace '{}': {e}", - branch.branch_name, - ws_name - ); - } - Ok(Value::Null) - } - Err(crate::blox::BloxError::NotAuthenticated) => { - store - .update_branch_workspace_status(&branch_id, &store::WorkspaceStatus::Error) - .ok(); - Err("Not authenticated with Blox. Run: sq login".to_string()) - } - Err(e) => { - if crate::branches::is_blox_onboarding_precondition_error(&e) { - store - .update_branch_workspace_status( - &branch_id, - &store::WorkspaceStatus::Error, - ) - .ok(); - return Err(e.to_string()); - } - log::warn!( - "blox ws start failed for '{}', leaving status as Starting for polling to resolve: {e}", - ws_name - ); - Ok(Value::Null) - } - } + crate::workspace_setup::ensure_branch_set_up( + &app_handle, + &store, + &branch_id, + crate::workspace_setup::SetupTrigger::Requested, + ) + .await?; + Ok(Value::Null) }) .await } @@ -2664,36 +2261,11 @@ pub(crate) async fn dispatch( let store = get_store(store_mutex)?; let workspace_name: String = arg(&args, "workspaceName")?; crate::project_transfer::ensure_workspace_not_transferring(&workspace_name)?; + let app_handle = app_handle.clone(); detached(async move { - let branch_ids = store - .update_workspace_status_by_workspace_name( - &workspace_name, - &store::WorkspaceStatus::Starting, - ) - .map_err(|e| e.to_string())?; - - if branch_ids.is_empty() { - return Err(format!("No branches found for workspace: {workspace_name}")); - } - - let ws = workspace_name.clone(); - if let Err(e) = - crate::branches::run_blox_blocking(move || crate::blox::ws_resume(&ws)).await - { - log::warn!( - "[resume_workspace] workspace={} resume failed: {}", - workspace_name, - e - ); - store - .update_workspace_status_by_workspace_name( - &workspace_name, - &store::WorkspaceStatus::Error, - ) - .ok(); - return Err(e.to_string()); - } - + let branch_ids = + crate::branches::resume_workspace_impl(&store, &app_handle, &workspace_name) + .await?; Ok(serde_json::to_value(branch_ids).unwrap()) }) .await @@ -5513,6 +5085,10 @@ mod tests { "resume_session_for_store(", "resume_or_queue_session_for_store(", "start_project_session_for_store(", + // The shared workspace bodies: branch setup, resume, the pollers. + "workspace_setup::ensure_branch_set_up(", + "resume_workspace_impl(", + "workspace_setup::poll::", ]; /// Arms whose bodies run through `detached`. const DETACHED: &[&str] = &[ @@ -5530,9 +5106,12 @@ mod tests { /// Arms that write only after their last await, with the kick in the /// same synchronous stretch: a drop before the write has written /// nothing (the polls redo it next tick), and nothing is left - /// half-written. + /// half-written. Each is one call into the shared poller of the same + /// name in `workspace_setup/poll.rs`, where that ordering is checked. const WRITES_AFTER_LAST_AWAIT: &[&str] = &["poll_workspace_status", "poll_all_workspace_statuses"]; + /// What the shared pollers write through, after their Blox call. + const POLL_WRITES: &[&str] = &["settle_one(", "settle_all("]; let source = include_str!("web_server.rs"); for (commands, arm) in dispatch_arms(source) { @@ -5554,17 +5133,31 @@ mod tests { .iter() .any(|c| WRITES_AFTER_LAST_AWAIT.contains(&c.as_str())) { - let last_await = arm.rfind(".await").expect("exempted arm should await"); - let first_write = WRITE_MARKERS + assert_eq!( + arm.matches(".await").count(), + 1, + "the web `{command}` arm must be one call into its shared poller: {arm}" + ); + let name = commands + .iter() + .find(|c| WRITES_AFTER_LAST_AWAIT.contains(&c.as_str())) + .unwrap(); + assert!( + arm.contains(&format!("workspace_setup::poll::{name}(")), + "the web `{command}` arm must call the shared poller: {arm}" + ); + let poller = shared_poller_body(name); + let last_await = poller.rfind(".await").expect("the poller should await"); + let first_write = POLL_WRITES .iter() - .filter_map(|marker| arm.find(marker)) + .filter_map(|marker| poller.find(marker)) .min() - .expect("exempted arm should write"); + .expect("the poller should write"); assert!( first_write > last_await, - "the web `{command}` arm writes before its last await; it no longer \ - qualifies for WRITES_AFTER_LAST_AWAIT and must run through \ - `detached(...)`" + "the shared `{name}` poller writes before its last await; the web \ + `{command}` arm no longer qualifies for WRITES_AFTER_LAST_AWAIT and \ + must run through `detached(...)`" ); } else { assert!( @@ -5578,6 +5171,18 @@ mod tests { } } + /// The body of the shared poller `name` in `workspace_setup/poll.rs`. + fn shared_poller_body(name: &str) -> &'static str { + let source = include_str!("workspace_setup/poll.rs"); + let start = source + .find(&format!("pub(crate) async fn {name}(")) + .unwrap_or_else(|| panic!("no shared poller named {name}")); + let end = source[start..] + .find("\n}\n") + .expect("the poller's body should end"); + &source[start..start + end] + } + /// The windowed transcript read must accept the same optional window the /// Tauri command does, and keep reading off the runtime thread: this is the /// largest payload the server produces, and the client pages through it diff --git a/apps/staged/src-tauri/src/workspace_setup.rs b/apps/staged/src-tauri/src/workspace_setup.rs new file mode 100644 index 000000000..1337557fc --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup.rs @@ -0,0 +1,543 @@ +//! Branch setup for remote (Blox) branches. +//! +//! A remote branch is usable once two things are true: its workstation is +//! running, and the workstation's clone is on the branch. Blox only knows +//! about the first. This module owns the second, and with it the one write +//! that says both hold: it is the only code that moves a branch from +//! `Starting` to `Running`, and it does so only after the checkout succeeded. +//! A failure leaves the branch in `Error` and is returned, never logged and +//! dropped — a branch marked `Running` on a workstation still on its base +//! branch has its sessions commit onto the base. +//! +//! Setup is level-triggered. Whoever sees a `Starting` branch asks for it: +//! +//! - the frontend, through the `start_workspace` command, for every +//! `Starting` branch it holds; +//! - the status pollers ([`poll`]), which see a running workstation whose +//! row is still `Starting` and trigger setup instead of promoting it; +//! - `resume_workspace`, once the workstation is back; +//! - the startup pass ([`resume_pending_setups`]), for branches a previous +//! run left in `Starting`. +//! +//! All of them go through one [`guard::SetupGuard`], so a second request for +//! a branch joins the run in flight, and two branches never run setup in one +//! workstation at once. The guard is per process: a second Staged instance +//! on the same database runs its own. + +pub(crate) mod guard; +pub(crate) mod poll; +#[cfg(test)] +mod tests; + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; + +use crate::blox::{self, BloxError}; +use crate::branches; +use crate::store::{self, Store, WorkspaceStatus}; + +use guard::SetupGuard; + +/// Who asked for a branch to be set up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SetupTrigger { + /// A caller that wants the branch set up whatever its row says, short of + /// `Running`: the `start_workspace` command, or a repo joining a running + /// workspace. + Requested, + /// Something that saw the row in `Starting` — a status poll, a resume, + /// the startup pass. It acts only while the row still says so, so a + /// trigger that arrives after setup settled the branch is a no-op. + Observed, +} + +/// How long setup waits for a workstation to report running after `sq blox +/// ws start` failed without saying why the workstation can't exist. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ReadyWait { + pub(crate) timeout: Duration, + pub(crate) interval: Duration, +} + +impl ReadyWait { + /// `ws start` itself gives up after 120 s while the workstation keeps + /// booting. Five more minutes covers a slow bootstrap; past that the + /// branch fails visibly rather than waiting on a workstation that may + /// never come. + pub(crate) const DEFAULT: ReadyWait = ReadyWait { + timeout: Duration::from_secs(300), + interval: Duration::from_secs(5), + }; +} + +/// The clone a branch is checked out in, and what to put it on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Checkout { + pub(crate) workspace: String, + /// `home:` from [`branches::resolve_branch_clone_dir`]; `None` for + /// a branch with no project repo, whose git runs in the workstation's + /// default directory. + pub(crate) clone_dir: Option, + pub(crate) repo_slug: String, + pub(crate) base_ref: String, + pub(crate) branch_name: String, +} + +/// Everything setup does outside the database, so tests drive the routine +/// without `sq`. [`SetupHost::check_out`] is where a test substitutes a +/// [`branches::GitRunner`] on a local clone for the workstation's. +#[async_trait] +pub(crate) trait SetupHost: Send + Sync + 'static { + /// Whether the user is signed in to Blox. Without it `ws start` can hang. + async fn check_auth(&self) -> Result<(), BloxError>; + + /// The status Blox reports for the workstation, as `ws info` words it. + async fn workspace_status(&self, workspace: &str) -> Result, BloxError>; + + /// `sq blox ws start `; returns once bootstrap is + /// done. + async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError>; + + /// Make sure the clone exists and put it on the branch + /// ([`branches::check_out_workspace_branch`]). + async fn check_out(&self, checkout: &Checkout) -> Result<(), String>; + + /// Called with what [`Store::update_branch_workspace_status`] reported + /// for the `Running` write (see [`branches::workspace_status_written`]). + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus); + + fn ready_wait(&self) -> ReadyWait { + ReadyWait::DEFAULT + } +} + +/// The real host: `sq blox` and the workstation's own git. +struct BloxHost { + app_handle: tauri::AppHandle, +} + +#[async_trait] +impl SetupHost for BloxHost { + async fn check_auth(&self) -> Result<(), BloxError> { + branches::run_blox_blocking(blox::check_auth).await + } + + async fn workspace_status(&self, workspace: &str) -> Result, BloxError> { + let name = workspace.to_string(); + let info = branches::run_blox_blocking(move || blox::ws_info(&name)).await?; + if let Some(ws_id) = info.workstation_id { + if let Ok(mut cache) = branches::workstation_id_cache().lock() { + cache.insert(workspace.to_string(), ws_id); + } + } + Ok(info.status) + } + + async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError> { + let name = workspace.to_string(); + let source = source.to_string(); + branches::run_blox_blocking(move || { + blox::ws_start( + &name, + Some(&source), + Some(branches::WORKSPACE_IDLE_TIMEOUT_MINUTES), + ) + }) + .await + .map(|_| ()) + } + + async fn check_out(&self, checkout: &Checkout) -> Result<(), String> { + match checkout.clone_dir.as_deref() { + Some(clone_dir) => { + branches::clone_repo_into_workspace( + &checkout.workspace, + clone_dir, + &checkout.repo_slug, + &checkout.base_ref, + &checkout.branch_name, + ) + .await + } + None => branches::check_out_branch_in_workspace( + &checkout.workspace, + None, + &checkout.base_ref, + &checkout.branch_name, + ) + .await + .map_err(|e| { + format!( + "Failed to set up branch '{}' in workspace '{}': {e}", + checkout.branch_name, checkout.workspace + ) + }), + } + } + + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { + branches::workspace_status_written(&self.app_handle, branch_id, changed, status); + } +} + +/// Whether Blox's status string means the workstation takes commands: the +/// same reading the pollers use, so a poll that triggers setup and the setup +/// it triggers agree on whether there is anything to start. +pub(crate) fn reports_running(blox_status: Option<&str>) -> bool { + blox_status.is_some() + && branches::map_blox_status_to_workspace_status( + blox_status, + Some(&WorkspaceStatus::Starting), + ) == WorkspaceStatus::Running +} + +/// Whether Blox says the workstation exists and is on its way up: being +/// created and bootstrapped, or resumed. It needs waiting for, not another +/// `ws start`. +fn reports_booting(blox_status: Option<&str>) -> bool { + blox_status.is_some_and(branches::is_blox_booting_status) +} + +/// Set `branch_id` up and mark it `Running`, joining a run already in flight +/// for it. The one entry point for every trigger. +pub(crate) async fn ensure_branch_set_up( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_id: &str, + trigger: SetupTrigger, +) -> Result<(), String> { + let host: Arc = Arc::new(BloxHost { + app_handle: app_handle.clone(), + }); + run_guarded(SetupGuard::global(), store, &host, branch_id, trigger).await +} + +/// Trigger setup for a branch seen in `Starting`, without waiting for it. +/// The outcome lands on the row: `Running`, or `Error`. +pub(crate) fn trigger_branch_setup( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_id: &str, +) { + // A poll every 3 s would otherwise spawn a task per tick just to join. + if SetupGuard::global().is_in_flight(branch_id) { + return; + } + let app_handle = app_handle.clone(); + let store = Arc::clone(store); + let branch_id = branch_id.to_string(); + tauri::async_runtime::spawn(async move { + if let Err(e) = + ensure_branch_set_up(&app_handle, &store, &branch_id, SetupTrigger::Observed).await + { + log::warn!("[workspace_setup] setup for branch {branch_id} failed: {e}"); + } + }); +} + +/// The startup pass: set up every remote branch a previous run left in +/// `Starting`, so recovery doesn't wait for a window to load the project. +/// +/// Call after interrupted transfers are recovered: a transfer's branches are +/// its own to provision. +pub(crate) fn resume_pending_setups( + app_handle: &tauri::AppHandle, + store: &Arc, +) { + let pending = match branches_awaiting_setup(store) { + Ok(pending) => pending, + Err(e) => { + log::warn!("[workspace_setup] could not list branches awaiting setup: {e}"); + return; + } + }; + if pending.is_empty() { + return; + } + log::info!( + "[workspace_setup] {} branch(es) left in Starting by a previous run", + pending.len() + ); + for branch in pending { + trigger_branch_setup(app_handle, store, &branch.id); + } +} + +/// Every remote branch whose row says `Starting`, skipping projects a +/// transfer is rewriting. +pub(crate) fn branches_awaiting_setup(store: &Store) -> Result, String> { + let mut pending = Vec::new(); + for project in store.list_projects().map_err(|e| e.to_string())? { + for branch in store + .list_branches_for_project(&project.id) + .map_err(|e| e.to_string())? + { + if branch.branch_type == store::BranchType::Remote + && branch.workspace_name.is_some() + && branch.workspace_status == Some(WorkspaceStatus::Starting) + && !crate::project_transfer::is_branch_transferring(&branch.id) + { + pending.push(branch); + } + } + } + Ok(pending) +} + +/// [`set_up_branch`] under `guard`: joined per branch, serialized per +/// workstation. +pub(crate) async fn run_guarded( + guard: &Arc, + store: &Arc, + host: &Arc, + branch_id: &str, + trigger: SetupTrigger, +) -> Result<(), String> { + let branch = store + .get_branch(branch_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + // Checked before taking a slot in the guard, so a trigger with nothing + // to do never becomes the run a real request joins. + if !wants_setup(&branch, trigger) { + return Ok(()); + } + let workspace = branch + .workspace_name + .clone() + .ok_or("Branch has no workspace name")?; + + let store = Arc::clone(store); + let host = Arc::clone(host); + let id = branch_id.to_string(); + guard + .run(&workspace, branch_id, move || async move { + set_up_branch(&store, host.as_ref(), &id, trigger).await + }) + .await +} + +/// Whether `trigger` has anything to do for a branch in this state. A +/// `Running` branch is set up already, and may have a session in its clone: +/// nothing touches it. An observed trigger acts only on `Starting`. +fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { + match (&branch.workspace_status, trigger) { + (Some(WorkspaceStatus::Running), _) => false, + (Some(WorkspaceStatus::Starting), SetupTrigger::Observed) => true, + (_, SetupTrigger::Observed) => false, + (_, SetupTrigger::Requested) => true, + } +} + +/// Bring a branch's workstation up, check the branch out in its clone, and +/// only then mark it `Running`. +/// +/// One routine for a workstation that has to be created and one that is +/// already running (a resumed workstation, a repo joining a shared one, a +/// retry after a restart). Both end in +/// [`branches::check_out_workspace_branch`], which is idempotent and never +/// resets a branch the clone already has. +/// +/// Every failure past the first lookups leaves the row in `Error` and is +/// returned. Callers go through [`run_guarded`]. +pub(crate) async fn set_up_branch( + store: &Arc, + host: &dyn SetupHost, + branch_id: &str, + trigger: SetupTrigger, +) -> Result<(), String> { + // A transfer provisions its own branches; refusing leaves the row alone. + crate::project_transfer::ensure_branch_not_transferring(branch_id)?; + + let branch = store + .get_branch(branch_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + // Read again now that the workstation's turn has come: the run ahead of + // this one may have settled the branch. + if !wants_setup(&branch, trigger) { + return Ok(()); + } + + // A request for a branch that had settled somewhere else (a failed + // setup, a stale `starting` in the window that asked) puts it back in + // `Starting`, where the pollers hold it rather than promote it under + // this run. + if branch.workspace_status != Some(WorkspaceStatus::Starting) { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) + .map_err(|e| e.to_string())?; + } + + let result = provision_and_check_out(store, host, &branch).await; + match &result { + Ok(()) => { + let changed = store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) + .map_err(|e| e.to_string())?; + host.status_written(branch_id, changed, &WorkspaceStatus::Running); + log::info!( + "[workspace_setup] branch={branch_id} checked out '{}' and running", + branch.branch_name + ); + } + Err(e) => { + log::warn!("[workspace_setup] branch={branch_id} setup failed: {e}"); + // Only over `Starting`. A poll that saw the workstation + // suspended or gone while this ran has already said something + // truer than `Error`, and with a way forward (Resume). + let still_starting = matches!( + store.get_branch(branch_id), + Ok(Some(row)) if row.workspace_status == Some(WorkspaceStatus::Starting) + ); + if still_starting { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Error) + .ok(); + } + } + } + result +} + +async fn provision_and_check_out( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, +) -> Result<(), String> { + let project = store + .get_project(&branch.project_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; + let workspace = branch + .workspace_name + .as_deref() + .ok_or("Branch has no workspace name")?; + let checkout = Checkout { + workspace: workspace.to_string(), + clone_dir: branches::resolve_branch_clone_dir(store, branch)?, + repo_slug: branches::resolve_branch_repo_slug(store, &project, branch)?, + base_ref: branches::normalize_branch_ref(&branch.base_branch), + branch_name: branch.branch_name.clone(), + }; + + // Pre-flight: without it `blox ws start` can hang or fail opaquely, + // leaving the frontend spinning for minutes (issue #99). + host.check_auth().await.map_err(|e| e.to_string())?; + + // A workstation that is already up — a resumed one, a shared one this + // repo joins, one a previous run started — is not started again, and + // neither is one that is on its way: `ws start` on a name that is + // already booting is at best a second bootstrap. A failed lookup is not + // an answer; `ws start` settles it. + let reported = host.workspace_status(workspace).await.ok().flatten(); + if reports_running(reported.as_deref()) { + // Nothing to start. + } else if reports_booting(reported.as_deref()) { + log::info!( + "[workspace_setup] branch={} workspace={workspace} is already booting, waiting for it", + branch.id + ); + wait_until_running(store, host, branch, workspace, "it was still booting").await?; + } else { + start_workstation(store, host, branch, &checkout).await?; + } + + host.check_out(&checkout).await +} + +/// `sq blox ws start`, and when that fails without ruling the workstation +/// out, a bounded wait for it to come up anyway. +async fn start_workstation( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, + checkout: &Checkout, +) -> Result<(), String> { + let workspace = checkout.workspace.as_str(); + let source = format!( + "https://github.com/{}.git?ref={}", + checkout.repo_slug, checkout.base_ref + ); + log::info!( + "[workspace_setup] branch={} workspace={workspace} starting from {source}", + branch.id + ); + let start_error = match host.start_workspace(workspace, &source).await { + Ok(()) => return Ok(()), + Err(BloxError::NotAuthenticated) => { + return Err("Not authenticated with Blox. Run: sq login".to_string()); + } + // Definitive: keep the CLI's text, which carries the onboarding URL + // and the action it wants. + Err(e) if branches::is_blox_onboarding_precondition_error(&e) => { + return Err(e.to_string()); + } + Err(e) => e, + }; + + // `ws start` can fail — its own 120 s limit, a dropped connection — after + // Blox created the workstation, which then keeps booting. Returning here + // would leave the branch with a running workstation and no checkout, so + // wait for it and carry on, or fail where the user can see it. + log::warn!( + "[workspace_setup] branch={} workspace={workspace} ws start failed, waiting for the workstation: {start_error}", + branch.id + ); + wait_until_running( + store, + host, + branch, + workspace, + &format!("`sq blox ws start` failed: {start_error}"), + ) + .await +} + +/// Wait, within [`SetupHost::ready_wait`], for the workstation to report +/// running. `why` says what the wait was for, for the error when it never +/// does. +async fn wait_until_running( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, + workspace: &str, + why: &str, +) -> Result<(), String> { + let wait = host.ready_wait(); + let started = tokio::time::Instant::now(); + loop { + match host.workspace_status(workspace).await { + Ok(status) if reports_running(status.as_deref()) => return Ok(()), + Ok(status) + if branches::map_blox_status_to_workspace_status( + status.as_deref(), + Some(&WorkspaceStatus::Starting), + ) == WorkspaceStatus::Error => + { + return Err(format!( + "Workspace '{workspace}' failed to start (Blox reports it as {}); {why}", + status.as_deref().unwrap_or("error") + )); + } + Err(BloxError::NotAuthenticated) => { + return Err("Not authenticated with Blox. Run: sq login".to_string()); + } + // Still booting, or not listed yet. + Ok(_) | Err(_) => {} + } + // Deleted while it waited: nothing left to set up. + if store.get_branch(&branch.id).ok().flatten().is_none() { + return Err(format!("Branch not found: {}", branch.id)); + } + if started.elapsed() >= wait.timeout { + return Err(format!( + "Workspace '{workspace}' did not come up within {}s; {why}", + wait.timeout.as_secs() + )); + } + tokio::time::sleep(wait.interval).await; + } +} diff --git a/apps/staged/src-tauri/src/workspace_setup/guard.rs b/apps/staged/src-tauri/src/workspace_setup/guard.rs new file mode 100644 index 000000000..b7aaefe5c --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/guard.rs @@ -0,0 +1,134 @@ +//! The in-process guard every branch setup runs under. +//! +//! Setup is asked for from several places at once — the frontend's +//! `start_workspace`, a status poll, a second window or web client, the +//! startup pass — and each of them means the same thing: "this branch should +//! be set up". Two checkouts in one clone collide on `.git/index.lock`, and +//! a caller that lost such a race used to report a failure for a branch that +//! was being set up fine. +//! +//! So a run is keyed by branch: a second caller for a branch whose setup is +//! in flight gets that run's result rather than starting its own. Runs for +//! different branches of one workstation take turns, since they may share a +//! clone. +//! +//! A run is its own task. A caller that goes away — a dropped web request, a +//! closed window — only stops waiting; the setup still finishes and still +//! writes the branch's status. + +use std::collections::HashMap; +use std::future::Future; +use std::sync::{Arc, Mutex, OnceLock}; + +use tokio::sync::watch; + +type Outcome = Option>; + +#[derive(Default)] +pub(crate) struct SetupGuard { + /// Branch id → the result of the run in flight for it, once it has one. + in_flight: Mutex>>, + /// Workspace name → the lock its runs take turns on. + workspace_locks: Mutex>>>, +} + +impl SetupGuard { + /// The guard the app's setups share. + pub(crate) fn global() -> &'static Arc { + static GUARD: OnceLock> = OnceLock::new(); + GUARD.get_or_init(|| Arc::new(SetupGuard::default())) + } + + /// Whether a run for `branch_id` has started and not yet finished. + pub(crate) fn is_in_flight(&self, branch_id: &str) -> bool { + self.lock_in_flight().contains_key(branch_id) + } + + /// Run `setup` for `branch_id` on `workspace`, or wait for the run + /// already in flight for that branch, and return its result. + /// + /// `setup` is not called when the caller joins. + pub(crate) async fn run( + self: &Arc, + workspace: &str, + branch_id: &str, + setup: F, + ) -> Result<(), String> + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let mut outcome = { + let mut in_flight = self.lock_in_flight(); + match in_flight.get(branch_id) { + Some(outcome) => outcome.clone(), + None => { + let (tx, rx) = watch::channel(None); + in_flight.insert(branch_id.to_string(), rx.clone()); + let turn = self.workspace_lock(workspace); + let entry = InFlight { + guard: Arc::clone(self), + branch_id: branch_id.to_string(), + }; + tauri::async_runtime::spawn(async move { + let result = { + let _turn = turn.lock().await; + setup().await + }; + // Cleared before the result is published: a caller + // that sees the result must be able to start a fresh + // run, not join a finished one. + drop(entry); + let _ = tx.send(Some(result)); + }); + rx + } + } + }; + + loop { + if let Some(result) = outcome.borrow_and_update().clone() { + return result; + } + // The sender is dropped without a result only if the run + // panicked; its entry is gone with it. + if outcome.changed().await.is_err() { + return Err("Workspace setup ended without a result".to_string()); + } + } + } + + fn lock_in_flight( + &self, + ) -> std::sync::MutexGuard<'_, HashMap>> { + self.in_flight.lock().unwrap_or_else(|p| p.into_inner()) + } + + fn workspace_lock(&self, workspace: &str) -> Arc> { + let mut locks = self + .workspace_locks + .lock() + .unwrap_or_else(|p| p.into_inner()); + // Drop the locks no run holds or waits on, so the map stays the size + // of what is being set up. + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks + .entry(workspace.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } +} + +/// A run's slot in [`SetupGuard::in_flight`], released when the run ends +/// however it ends — a panic included, or the branch could never be set up +/// again in this process. +struct InFlight { + guard: Arc, + branch_id: String, +} + +impl Drop for InFlight { + fn drop(&mut self) { + self.guard.lock_in_flight().remove(&self.branch_id); + } +} diff --git a/apps/staged/src-tauri/src/workspace_setup/poll.rs b/apps/staged/src-tauri/src/workspace_setup/poll.rs new file mode 100644 index 000000000..6881bcfad --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/poll.rs @@ -0,0 +1,371 @@ +//! The workspace status pollers, shared by the Tauri commands and the web +//! server's arms. +//! +//! A poll reconciles a branch row with what Blox reports for its +//! workstation. It owns every transition but one: it never writes `Running` +//! over `Starting`. A running workstation says nothing about whether the +//! branch is checked out in it, so a poll that finds one behind a `Starting` +//! row triggers branch setup (see the parent module) and reports `Starting` +//! until setup has written `Running` itself. Nor does it write `Running` +//! over `Error`: a setup that failed left the row there, and a client still +//! polling on a stale `starting` must be told so, not handed a branch whose +//! checkout never happened. + +use std::collections::HashMap; +use std::sync::Arc; + +use tauri::AppHandle; + +use crate::blox::{self, BloxError}; +use crate::branches; +use crate::store::{self, Store, WorkspaceStatus}; +use crate::PollWorkspaceResult; + +/// Called with a branch id and what [`Store::update_branch_workspace_status`] +/// reported for a status write (see [`branches::workspace_status_written`]). +pub(crate) type StatusWritten<'a> = Box; + +/// What a poll does besides writing the row, injected so tests can watch it. +pub(crate) struct PollEffects<'a> { + pub(crate) status_written: StatusWritten<'a>, + /// Called for a `Starting` branch whose workstation is running. + pub(crate) trigger_setup: Box, +} + +/// What Blox said about one workstation. +pub(crate) struct Reported<'a> { + pub(crate) status: Option<&'a str>, + pub(crate) workstation_id: Option, +} + +fn result(status: &WorkspaceStatus, workspace: &str) -> PollWorkspaceResult { + PollWorkspaceResult { + status: status.as_str().to_string(), + workstation_id: branches::cached_workstation_id(workspace), + } +} + +/// Whether `branch` is a secondary clone still being set up in a workstation +/// a peer branch already runs on. Such a branch uses `Starting` as its own +/// loading state while its clone, fetch and checkout run, and is held there +/// until that setup marks it `Running`. +/// +/// This must apply only to secondary clones. A project's first remote branch +/// has a workspace subpath too, so keying on the subpath alone would skip +/// polling for it altogether. +pub(crate) fn is_secondary_clone_setup( + store: &Store, + branch: &store::Branch, +) -> Result { + if branch.workspace_status != Some(WorkspaceStatus::Starting) + || branches::resolve_branch_workspace_subpath(store, branch)?.is_none() + { + return Ok(false); + } + let Some(workspace) = branch.workspace_name.as_deref() else { + return Ok(false); + }; + let peers = store + .list_branches_for_project(&branch.project_id) + .map_err(|e| e.to_string())?; + Ok(peers.into_iter().any(|peer| { + peer.id != branch.id + && peer.branch_type == store::BranchType::Remote + && peer.workspace_name.as_deref() == Some(workspace) + && peer.workspace_status == Some(WorkspaceStatus::Running) + })) +} + +/// Reconcile `branch`'s row with the status Blox reported, and return the +/// status the row now holds. +pub(crate) fn settle_reported( + store: &Store, + branch: &store::Branch, + workspace: &str, + reported: &Reported<'_>, + effects: &PollEffects<'_>, +) -> Result { + // Cache the numeric workstation ID for proxy URL construction. + if let Some(ws_id) = reported.workstation_id { + if let Ok(mut cache) = branches::workstation_id_cache().lock() { + cache.insert(workspace.to_string(), ws_id); + } + } + + let row = branch.workspace_status.as_ref(); + let mapped = branches::map_blox_status_to_workspace_status(reported.status, row); + if mapped == WorkspaceStatus::Running { + match row { + // The workstation is up; the branch may not be checked out in + // it. Setup finds out, and is the one to say `Running`. + Some(WorkspaceStatus::Starting) => { + (effects.trigger_setup)(&branch.id); + return Ok(WorkspaceStatus::Starting); + } + // Setup failed here. Only another setup can clear that. + Some(WorkspaceStatus::Error) => return Ok(WorkspaceStatus::Error), + _ => {} + } + } + + let previous = row.map(|s| s.as_str()).unwrap_or("none"); + if previous != mapped.as_str() { + log::debug!( + "[workspace_poll] branch={} ws={workspace} transition {previous} -> {} (raw={:?})", + branch.id, + mapped.as_str(), + reported.status + ); + } + let changed = store + .update_branch_workspace_status(&branch.id, &mapped) + .map_err(|e| e.to_string())?; + (effects.status_written)(&branch.id, changed, &mapped); + Ok(mapped) +} + +/// Settle one branch against the outcome of its `ws info`. +pub(crate) fn settle_one( + store: &Store, + branch: &store::Branch, + workspace: &str, + info: Result, &BloxError>, + effects: &PollEffects<'_>, +) -> Result { + let reported = match info { + Ok(reported) => reported, + Err(BloxError::NotAuthenticated) => { + // Auth errors are definitive — stop polling and surface the error. + store + .update_branch_workspace_status(&branch.id, &WorkspaceStatus::Error) + .ok(); + return Err("Not authenticated with Blox. Run: sq login".to_string()); + } + Err(e) => { + // During initial creation, `blox ws start` may still be running + // when the first poll fires `blox ws info`. The workspace doesn't + // exist yet, so the CLI returns "not found". If the DB still says + // Starting, swallow the error and keep polling. + if branch.workspace_status == Some(WorkspaceStatus::Starting) { + log::debug!( + "blox ws info failed for '{workspace}' while still Starting, treating as Starting: {e}" + ); + return Ok(result(&WorkspaceStatus::Starting, workspace)); + } + // If the workspace was Running but Blox explicitly says "not + // found" (e.g. deleted externally), transition to Stopped. Other + // errors (network blips, timeouts, etc.) are transient — keep the + // Running status so the poller retries. + if branch.workspace_status == Some(WorkspaceStatus::Running) { + let is_not_found = matches!(e, BloxError::CommandFailed(msg) if msg.to_lowercase().contains("not found")); + if is_not_found { + log::debug!( + "blox ws info returned not-found for '{workspace}' while Running, treating as Stopped: {e}" + ); + store + .update_branch_workspace_status(&branch.id, &WorkspaceStatus::Stopped) + .ok(); + return Ok(result(&WorkspaceStatus::Stopped, workspace)); + } + log::warn!( + "blox ws info failed for '{workspace}' while Running, keeping Running status: {e}" + ); + return Ok(result(&WorkspaceStatus::Running, workspace)); + } + return Err(e.to_string()); + } + }; + + let status = settle_reported(store, branch, workspace, &reported, effects)?; + Ok(result(&status, workspace)) +} + +/// What a batch poll settled. +pub(crate) struct BatchSettled { + pub(crate) results: HashMap, + /// Workspace name → the branches still `Starting` on it, whose bootstrap + /// progress is worth fetching. + pub(crate) starting_workspaces: HashMap>, +} + +/// Settle `branch_ids` against one `ws list`. Branches that cannot be +/// resolved (not in the store, no workspace name) are left out. +pub(crate) fn settle_all( + store: &Store, + branch_ids: &[String], + entries: &[blox::WorkspaceListEntry], + effects: &PollEffects<'_>, +) -> BatchSettled { + let listed: HashMap<&str, &blox::WorkspaceListEntry> = + entries.iter().map(|e| (e.name.as_str(), e)).collect(); + let mut settled = BatchSettled { + results: HashMap::new(), + starting_workspaces: HashMap::new(), + }; + + for branch_id in branch_ids { + let Ok(Some(branch)) = store.get_branch(branch_id) else { + continue; + }; + let Some(workspace) = branch.workspace_name.as_deref() else { + continue; + }; + + let held = is_secondary_clone_setup(store, &branch).unwrap_or_else(|e| { + log::warn!("Failed to check secondary clone setup for branch {branch_id}: {e}"); + false + }); + if held { + log::debug!( + "[workspace_poll] branch={branch_id} ws={workspace} held at Starting for secondary clone setup" + ); + settled.results.insert( + branch_id.clone(), + result(&WorkspaceStatus::Starting, workspace), + ); + continue; + } + + let status = match listed.get(workspace) { + Some(entry) => { + let reported = Reported { + status: entry.status.as_deref(), + workstation_id: entry.workstation_id, + }; + match settle_reported(store, &branch, workspace, &reported, effects) { + Ok(status) => status, + Err(e) => { + log::warn!("[workspace_poll] branch={branch_id} status write failed: {e}"); + continue; + } + } + } + // Not in the list — the same "not found" reading as a single poll. + None => match branch.workspace_status { + Some(WorkspaceStatus::Starting) => WorkspaceStatus::Starting, + Some(WorkspaceStatus::Running) => { + log::debug!( + "[workspace_poll] branch={branch_id} ws={workspace} not in list while Running, treating as Stopped" + ); + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Stopped) + .ok(); + WorkspaceStatus::Stopped + } + // Stopped, Error, etc. — no change, nothing to report. + _ => continue, + }, + }; + + // Track workspaces still starting so their bootstrap progress shows. + if status == WorkspaceStatus::Starting { + settled + .starting_workspaces + .entry(workspace.to_string()) + .or_default() + .push(branch_id.clone()); + } + settled + .results + .insert(branch_id.clone(), result(&status, workspace)); + } + + settled +} + +/// The `poll_workspace_status` command and web arm. +pub(crate) async fn poll_workspace_status( + store: &Arc, + app_handle: &AppHandle, + branch_id: &str, +) -> Result { + let branch = store + .get_branch(branch_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + let workspace = branch + .workspace_name + .clone() + .ok_or("Branch is not a remote workspace branch")?; + + if is_secondary_clone_setup(store, &branch)? { + log::debug!( + "[workspace_poll] branch={branch_id} ws={workspace} held at Starting for secondary clone setup" + ); + return Ok(result(&WorkspaceStatus::Starting, &workspace)); + } + + let info = branches::run_blox_blocking({ + let workspace = workspace.clone(); + move || blox::ws_info(&workspace) + }) + .await; + let reported = info.as_ref().map(|info| Reported { + status: info.status.as_deref(), + workstation_id: info.workstation_id, + }); + settle_one( + store, + &branch, + &workspace, + reported, + &app_effects(store, app_handle), + ) +} + +/// The `poll_all_workspace_statuses` command and web arm. +pub(crate) async fn poll_all_workspace_statuses( + store: &Arc, + app_handle: &AppHandle, + branch_ids: &[String], +) -> Result, String> { + // Fetch all workspaces in one CLI call. + let entries = branches::run_blox_blocking(blox::ws_list) + .await + .map_err(|e| { + if matches!(e, BloxError::NotAuthenticated) { + "Not authenticated with Blox. Run: sq login".to_string() + } else { + e.to_string() + } + })?; + + let BatchSettled { + results, + starting_workspaces, + } = settle_all(store, branch_ids, &entries, &app_effects(store, app_handle)); + + // For workspaces that are still starting, fetch bootstrap command + // progress and emit events so the UI can show which step is running. On + // its own blocking task, so it doesn't hold up the response. + if !starting_workspaces.is_empty() { + let app_handle = app_handle.clone(); + tauri::async_runtime::spawn_blocking(move || { + for (workspace, branch_ids) in &starting_workspaces { + match blox::ws_commands(workspace) { + Ok(commands) => { + branches::emit_workspace_setup_progress(&app_handle, branch_ids, &commands); + } + Err(e) => { + log::debug!("[workspace_poll] ws_commands({workspace}) failed: {e}"); + } + } + } + }); + } + + Ok(results) +} + +/// The effects a real poll has: kick the session queue on a `Running` write, +/// and hand a `Starting` branch on a running workstation to setup. +fn app_effects<'a>(store: &'a Arc, app_handle: &'a AppHandle) -> PollEffects<'a> { + PollEffects { + status_written: Box::new(move |branch_id, changed, status| { + branches::workspace_status_written(app_handle, branch_id, changed, status); + }), + trigger_setup: Box::new(move |branch_id| { + super::trigger_branch_setup(app_handle, store, branch_id); + }), + } +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests.rs b/apps/staged/src-tauri/src/workspace_setup/tests.rs new file mode 100644 index 000000000..a454f6446 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests.rs @@ -0,0 +1,1005 @@ +//! Branch setup, its guard, the pollers' share of it and the startup pass, +//! driven without `sq`: a [`FakeHost`] answers for Blox, and a local clone +//! of a bare origin stands in for the workstation's, reached through the +//! same [`branches::GitRunner`] the real checkout runs on. + +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use tokio::sync::Semaphore; + +use super::guard::SetupGuard; +use super::poll::{self, PollEffects, Reported}; +use super::{branches_awaiting_setup, run_guarded, Checkout, ReadyWait, SetupHost, SetupTrigger}; +use crate::blox::{BloxError, WorkspaceListEntry}; +use crate::branches; +use crate::store::{self, Store, WorkspaceStatus}; +use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; + +const WORKSPACE: &str = "widgets-setup-test"; + +// ── Fixtures ──────────────────────────────────────────────────────────────── + +/// A bare origin with `main`, and the clone Blox's bootstrap leaves behind: +/// on `main`, nothing else checked out. +fn origin_and_workstation_clone() -> (TempDir, TempDir) { + let seed = TempGitRepo::new(); + seed.write_file("file.txt", "base\n"); + seed.commit("base"); + let origin = TempDir::new("origin"); + run_git_in( + seed.path(), + &["clone", "--bare", ".", origin.path().to_str().unwrap()], + ); + let clone = TempDir::new("workstation"); + run_git_in( + origin.path(), + &[ + "clone", + origin.path().to_str().unwrap(), + clone.path().to_str().unwrap(), + ], + ); + run_git_in(clone.path(), &["config", "user.email", "test@example.com"]); + run_git_in(clone.path(), &["config", "user.name", "Test"]); + (origin, clone) +} + +fn head_branch(clone: &Path) -> String { + run_git_in(clone, &["rev-parse", "--abbrev-ref", "HEAD"]) + .trim() + .to_string() +} + +fn store_with_project() -> (Arc, store::Project) { + let store = Arc::new(Store::in_memory().unwrap()); + let project = store::Project::new("acme/widgets"); + store.create_project(&project).unwrap(); + (store, project) +} + +/// A remote branch in `Starting`, the way `create_remote_branch` leaves it. +fn starting_branch(store: &Store, project: &store::Project, name: &str) -> store::Branch { + let branch = store::Branch::new_remote(&project.id, name, "main", WORKSPACE); + store.create_branch(&branch).unwrap(); + branch +} + +fn status_of(store: &Store, branch: &store::Branch) -> Option { + store + .get_branch(&branch.id) + .unwrap() + .unwrap() + .workspace_status +} + +fn set_status(store: &Store, branch: &store::Branch, status: WorkspaceStatus) -> store::Branch { + store + .update_branch_workspace_status(&branch.id, &status) + .unwrap(); + store.get_branch(&branch.id).unwrap().unwrap() +} + +/// What the fake workstation answers a status lookup with. +#[derive(Clone, Copy)] +enum Info { + Status(&'static str), + /// `ws info` failed: the workstation isn't listed yet, or the call + /// dropped. + Unreachable, +} + +#[derive(Clone, Copy)] +enum Start { + Succeeds, + /// Exits non-zero or times out, with nothing about auth or onboarding. + Fails(&'static str), + NotAuthenticated, +} + +/// Blox and the workstation's git, faked. +struct FakeHost { + clone: PathBuf, + /// Answers to successive status lookups; the last one repeats. + info: Mutex>, + start: Start, + checkout_error: Option<&'static str>, + /// When set, a checkout waits for a permit before touching the clone. + gate: Option>, + /// Run at each status lookup: what else happens while setup is out. + meanwhile: Option>, + calls: Mutex>, + written: Mutex>, + in_checkout: AtomicUsize, + most_in_checkout: AtomicUsize, +} + +impl FakeHost { + fn new(clone: &Path, info: &[Info], start: Start) -> Self { + Self { + clone: clone.to_path_buf(), + info: Mutex::new(info.iter().copied().collect()), + start, + checkout_error: None, + gate: None, + meanwhile: None, + calls: Mutex::new(Vec::new()), + written: Mutex::new(Vec::new()), + in_checkout: AtomicUsize::new(0), + most_in_checkout: AtomicUsize::new(0), + } + } + + fn gated(mut self) -> (Self, Arc) { + let gate = Arc::new(Semaphore::new(0)); + self.gate = Some(Arc::clone(&gate)); + (self, gate) + } + + fn count(&self, call: &str) -> usize { + self.calls + .lock() + .unwrap() + .iter() + .filter(|c| **c == call) + .count() + } + + fn written(&self) -> Vec<(String, bool, WorkspaceStatus)> { + self.written.lock().unwrap().clone() + } + + async fn wait_for_checkouts(&self, count: usize) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while self.count("check_out") < count { + assert!( + std::time::Instant::now() < deadline, + "checkout never started" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } +} + +#[async_trait] +impl SetupHost for FakeHost { + async fn check_auth(&self) -> Result<(), BloxError> { + self.calls.lock().unwrap().push("check_auth"); + Ok(()) + } + + async fn workspace_status(&self, _workspace: &str) -> Result, BloxError> { + self.calls.lock().unwrap().push("workspace_status"); + if let Some(meanwhile) = &self.meanwhile { + meanwhile(); + } + let info = { + let mut info = self.info.lock().unwrap(); + if info.len() > 1 { + info.pop_front().unwrap() + } else { + *info.front().unwrap() + } + }; + match info { + Info::Status(status) => Ok(Some(status.to_string())), + Info::Unreachable => Err(BloxError::CommandFailed("workspace not found".into())), + } + } + + async fn start_workspace(&self, _workspace: &str, _source: &str) -> Result<(), BloxError> { + self.calls.lock().unwrap().push("start_workspace"); + match self.start { + Start::Succeeds => Ok(()), + Start::Fails(message) => Err(BloxError::CommandFailed(message.to_string())), + Start::NotAuthenticated => Err(BloxError::NotAuthenticated), + } + } + + async fn check_out(&self, checkout: &Checkout) -> Result<(), String> { + self.calls.lock().unwrap().push("check_out"); + if let Some(gate) = &self.gate { + gate.acquire().await.unwrap().forget(); + } + let now = self.in_checkout.fetch_add(1, Ordering::SeqCst) + 1; + self.most_in_checkout.fetch_max(now, Ordering::SeqCst); + // Yield with the clone "busy", so a second checkout let in beside + // this one would be counted. + tokio::time::sleep(Duration::from_millis(20)).await; + let result = match self.checkout_error { + Some(error) => Err(error.to_string()), + None => branches::check_out_workspace_branch( + &branches::git_runner(&self.clone, None, None), + &checkout.base_ref, + &checkout.branch_name, + ), + }; + self.in_checkout.fetch_sub(1, Ordering::SeqCst); + result + } + + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { + self.written + .lock() + .unwrap() + .push((branch_id.to_string(), changed, status.clone())); + } + + fn ready_wait(&self) -> ReadyWait { + ReadyWait { + timeout: Duration::from_millis(200), + interval: Duration::from_millis(5), + } + } +} + +async fn set_up( + store: &Arc, + host: &Arc, + branch: &store::Branch, + trigger: SetupTrigger, +) -> Result<(), String> { + let guard = Arc::new(SetupGuard::default()); + let host: Arc = host.clone(); + run_guarded(&guard, store, &host, &branch.id, trigger).await +} + +// ── The setup routine ─────────────────────────────────────────────────────── + +/// The run that left a workstation on `main`: `sq blox ws start` fails after +/// Blox created the workstation, which goes on to boot. Setup used to return +/// success there and leave the checkout undone. It now waits for the +/// workstation and checks the branch out before saying `Running`. +#[tokio::test] +async fn a_failed_ws_start_still_ends_in_a_checked_out_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[ + Info::Unreachable, + Info::Status("creating"), + Info::Status("starting"), + Info::Status("running"), + ], + Start::Fails("sq blox command timed out after 120s"), + )); + + set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!( + host.written(), + vec![(branch.id.clone(), true, WorkspaceStatus::Running)] + ); + assert_eq!(host.count("start_workspace"), 1); +} + +/// When the workstation never comes up after a failed start, the branch +/// fails where the user can see it, with the start error, instead of +/// sitting in `Starting`. +#[tokio::test] +async fn a_failed_ws_start_with_no_workstation_fails_visibly() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Unreachable, Info::Status("creating")], + Start::Fails("quota exceeded"), + )); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("did not come up"), "{error}"); + assert!(error.contains("quota exceeded"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.count("check_out"), 0); + assert_eq!(host.written(), vec![]); + assert_eq!(head_branch(clone.path()), "main"); +} + +/// A workstation Blox reports as failed ends the wait at once. +#[tokio::test] +async fn a_workstation_that_fails_to_boot_ends_the_wait() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Unreachable, Info::Status("failed")], + Start::Fails("connection reset"), + )); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("failed to start"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + // The first lookup, then the one that saw `failed`: no waiting it out. + assert_eq!(host.count("workspace_status"), 2); +} + +/// A relaunch can find the workstation a previous run started still booting, +/// and a resume finds it coming back. Either way it is waited for: a second +/// `ws start` on that name is never sent. +#[tokio::test] +async fn a_booting_workstation_is_waited_for_not_started_again() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[ + Info::Status("starting"), + Info::Status("starting"), + Info::Status("running"), + ], + Start::Fails("workspace already exists"), + )); + + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// ...and one that never arrives is a visible failure too. +#[tokio::test] +async fn a_booting_workstation_that_never_arrives_fails_visibly() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("provisioning")], + Start::Succeeds, + )); + + let error = set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap_err(); + + assert!(error.contains("still booting"), "{error}"); + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(host.count("check_out"), 0); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); +} + +/// A poll that saw the workstation suspended while setup waited has said +/// something truer than `Error`, and the branch keeps its Resume button. The +/// caller still gets the failure. +#[tokio::test] +async fn a_failed_setup_does_not_overwrite_what_a_poll_settled() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let mut host = FakeHost::new( + clone.path(), + &[Info::Status("suspended")], + Start::Fails("workspace is suspended"), + ); + host.meanwhile = Some(Box::new({ + let (store, id) = (Arc::clone(&store), branch.id.clone()); + move || { + store + .update_branch_workspace_status(&id, &WorkspaceStatus::Suspended) + .unwrap(); + } + })); + let host = Arc::new(host); + + let error = set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap_err(); + + assert!(error.contains("did not come up"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Suspended)); +} + +/// A request for a branch whose last setup failed is a retry. The row goes +/// back to `Starting` for its length, where the pollers hold it, and ends +/// `Running` once the checkout is done. +#[tokio::test] +async fn a_request_retries_a_failed_branch_from_starting() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Error); + let during = Arc::new(Mutex::new(Vec::new())); + let mut host = FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds); + host.meanwhile = Some(Box::new({ + let (store, id, during) = (Arc::clone(&store), branch.id.clone(), Arc::clone(&during)); + move || { + let row = store.get_branch(&id).unwrap().unwrap(); + during.lock().unwrap().push(row.workspace_status); + } + })); + let host = Arc::new(host); + + set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!( + *during.lock().unwrap(), + vec![Some(WorkspaceStatus::Starting)] + ); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// Auth failures stay definitive: no wait, and the message says what to do. +#[tokio::test] +async fn an_unauthenticated_ws_start_is_not_waited_out() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Unreachable], + Start::NotAuthenticated, + )); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert_eq!(error, "Not authenticated with Blox. Run: sq login"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.count("workspace_status"), 1); +} + +/// A checkout that fails is the command's error and the branch's `Error`. +/// It used to be a log line under a success. +#[tokio::test] +async fn a_failed_checkout_is_returned_and_never_marked_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let mut host = FakeHost::new(clone.path(), &[Info::Unreachable], Start::Succeeds); + host.checkout_error = Some("could not create branch 'feature-new': index.lock exists"); + let host = Arc::new(host); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("index.lock"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.written(), vec![]); +} + +/// A fresh workstation and one that is already up end in the same checkout; +/// only the second skips `ws start`. +#[tokio::test] +async fn fresh_and_running_workstations_share_one_checkout() { + for (info, starts) in [(Info::Unreachable, 1), (Info::Status("running"), 0)] { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new(clone.path(), &[info], Start::Succeeds)); + + set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!(host.count("start_workspace"), starts); + assert_eq!(host.count("check_out"), 1); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + } +} + +/// Resume puts the branch back in `Starting` on a workstation that already +/// has it, unpushed commits and all. Setup ends with it checked out and +/// `Running`, and the commits still there. +#[tokio::test] +async fn a_resumed_branch_ends_checked_out_with_its_commits() { + let (_origin, clone) = origin_and_workstation_clone(); + run_git_in(clone.path(), &["checkout", "-b", "feature-new"]); + std::fs::write(clone.path().join("file.txt"), "base\nunpushed\n").unwrap(); + run_git_in(clone.path(), &["commit", "-am", "unpushed"]); + let unpushed = run_git_in(clone.path(), &["rev-parse", "HEAD"]); + // Another branch of the repo had the shared clone last. + run_git_in(clone.path(), &["checkout", "main"]); + + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Suspended); + store + .update_workspace_status_by_workspace_name(WORKSPACE, &WorkspaceStatus::Starting) + .unwrap(); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(run_git_in(clone.path(), &["rev-parse", "HEAD"]), unpushed); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A `Running` branch may have a session in its clone: no trigger touches +/// it. An observed trigger also leaves alone anything that isn't `Starting`. +#[tokio::test] +async fn a_settled_branch_is_left_alone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + let running = set_status(&store, &branch, WorkspaceStatus::Running); + for trigger in [SetupTrigger::Requested, SetupTrigger::Observed] { + set_up(&store, &host, &running, trigger).await.unwrap(); + } + let failed = set_status(&store, &branch, WorkspaceStatus::Error); + set_up(&store, &host, &failed, SetupTrigger::Observed) + .await + .unwrap(); + + assert_eq!(*host.calls.lock().unwrap(), Vec::<&str>::new()); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(head_branch(clone.path()), "main"); +} + +/// A fetch that never reached origin is a failure. Read as "the branch is +/// not on origin", it would create the branch from the base and leave a PR's +/// commits behind. +#[test] +fn a_failed_origin_probe_is_not_read_as_a_new_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let real = branches::git_runner(clone.path(), None, None); + let git: branches::GitRunner<'_> = Box::new(move |args: &[&str]| { + if args.first() == Some(&"ls-remote") { + return Err("sq blox command timed out after 300s".to_string()); + } + real(args) + }); + + let error = branches::check_out_workspace_branch(&git, "main", "feature-new").unwrap_err(); + + assert!(error.contains("is on origin"), "{error}"); + assert_eq!(head_branch(clone.path()), "main"); +} + +// ── The guard ─────────────────────────────────────────────────────────────── + +/// The frontend's call, a poll's trigger, a second window and the startup +/// pass all asking for one branch get one setup, and all of them its result. +#[tokio::test] +async fn concurrent_callers_join_one_setup() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let (host, gate) = + FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds).gated(); + let host = Arc::new(host); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + let callers: Vec<_> = [ + SetupTrigger::Requested, + SetupTrigger::Observed, + SetupTrigger::Requested, + SetupTrigger::Observed, + ] + .into_iter() + .map(|trigger| { + let (guard, store, host, id) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&dyn_host), + branch.id.clone(), + ); + tokio::spawn(async move { run_guarded(&guard, &store, &host, &id, trigger).await }) + }) + .collect(); + + host.wait_for_checkouts(1).await; + assert!(guard.is_in_flight(&branch.id)); + // Room for every caller to run its own checkout, were it going to. + gate.add_permits(4); + for caller in callers { + assert_eq!(caller.await.unwrap(), Ok(())); + } + + assert_eq!(host.count("check_out"), 1); + assert_eq!(host.written().len(), 1); + assert!(!guard.is_in_flight(&branch.id)); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(head_branch(clone.path()), "feature-new"); +} + +/// A caller that joins a failing setup gets the failure it joined, not one +/// of its own from colliding with it. +#[tokio::test] +async fn joined_callers_share_a_failure() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let mut host = FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds); + host.checkout_error = Some("could not fetch base branch 'main': network"); + let (host, gate) = host.gated(); + let host = Arc::new(host); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + let callers: Vec<_> = (0..2) + .map(|_| { + let (guard, store, host, id) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&dyn_host), + branch.id.clone(), + ); + tokio::spawn(async move { + run_guarded(&guard, &store, &host, &id, SetupTrigger::Requested).await + }) + }) + .collect(); + host.wait_for_checkouts(1).await; + gate.add_permits(2); + + for caller in callers { + let error = caller.await.unwrap().unwrap_err(); + assert!(error.contains("could not fetch base branch"), "{error}"); + } + assert_eq!(host.count("check_out"), 1); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); +} + +/// Two branches of one workstation can share a clone, so their setups take +/// turns rather than meeting on `.git/index.lock`. +#[tokio::test] +async fn setups_in_one_workstation_take_turns() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let first = starting_branch(&store, &project, "feature-one"); + let second = starting_branch(&store, &project, "feature-two"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + let (a, b) = tokio::join!( + run_guarded( + &guard, + &store, + &dyn_host, + &first.id, + SetupTrigger::Requested + ), + run_guarded( + &guard, + &store, + &dyn_host, + &second.id, + SetupTrigger::Requested + ), + ); + + assert_eq!((a, b), (Ok(()), Ok(()))); + assert_eq!(host.count("check_out"), 2); + assert_eq!(host.most_in_checkout.load(Ordering::SeqCst), 1); + assert_eq!(status_of(&store, &first), Some(WorkspaceStatus::Running)); + assert_eq!(status_of(&store, &second), Some(WorkspaceStatus::Running)); +} + +/// A setup that panics gives its callers an error and frees the branch for +/// the next attempt. +#[tokio::test] +async fn a_panicked_setup_releases_the_branch() { + let guard = Arc::new(SetupGuard::default()); + + let outcome = guard + .run(WORKSPACE, "branch", || async { panic!("setup blew up") }) + .await; + assert!(outcome.is_err()); + assert!(!guard.is_in_flight("branch")); + + let retry = guard.run(WORKSPACE, "branch", || async { Ok(()) }).await; + assert_eq!(retry, Ok(())); +} + +// ── The pollers ───────────────────────────────────────────────────────────── + +/// What a poll did besides writing the row. +#[derive(Default)] +struct Seen { + written: Mutex>, + triggered: Mutex>, +} + +impl Seen { + fn effects(&self) -> PollEffects<'_> { + PollEffects { + status_written: Box::new(|branch_id, changed, status| { + self.written + .lock() + .unwrap() + .push((branch_id.to_string(), changed, status.clone())); + }), + trigger_setup: Box::new(|branch_id| { + self.triggered.lock().unwrap().push(branch_id.to_string()); + }), + } + } + + fn triggered(&self) -> Vec { + self.triggered.lock().unwrap().clone() + } + + fn written(&self) -> Vec<(String, bool, WorkspaceStatus)> { + self.written.lock().unwrap().clone() + } +} + +fn listed(status: &str) -> Vec { + vec![WorkspaceListEntry { + name: WORKSPACE.to_string(), + workstation_id: Some(42), + status: Some(status.to_string()), + extra: serde_json::Value::Null, + }] +} + +fn poll_batch(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { + let settled = poll::settle_all( + store, + &[branch.id.clone()], + &listed(blox_status), + &seen.effects(), + ); + settled.results[&branch.id].status.clone() +} + +fn poll_single(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { + poll::settle_one( + store, + branch, + WORKSPACE, + Ok(Reported { + status: Some(blox_status), + workstation_id: Some(42), + }), + &seen.effects(), + ) + .unwrap() + .status +} + +/// The other half of the bug: a poll saw the workstation running and marked +/// the branch `Running` with nothing checked out. Both pollers now leave the +/// row in `Starting` and ask for setup. +#[test] +fn a_poll_triggers_setup_instead_of_promoting_a_starting_branch() { + for poll in [poll_batch, poll_single] { + for blox_status in ["running", "ready", "active"] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let seen = Seen::default(); + + let reported = poll(&store, &branch, blox_status, &seen); + + assert_eq!(reported, "starting"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); + assert_eq!(seen.triggered(), vec![branch.id.clone()]); + // Nothing was written, so no session queue was kicked. + assert_eq!(seen.written(), vec![]); + } + } +} + +/// A batch poll also keeps fetching bootstrap progress for such a branch. +#[test] +fn a_batch_poll_keeps_a_branch_awaiting_setup_among_the_starting() { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let seen = Seen::default(); + + let settled = poll::settle_all( + &store, + &[branch.id.clone()], + &listed("running"), + &seen.effects(), + ); + + assert_eq!( + settled.starting_workspaces.get(WORKSPACE), + Some(&vec![branch.id.clone()]) + ); +} + +/// A setup that failed left the row in `Error`. A client still polling on a +/// stale `starting` is told so, rather than handed a `Running` branch whose +/// checkout never happened. +#[test] +fn a_poll_does_not_clear_a_failed_setup() { + for poll in [poll_batch, poll_single] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Error); + let seen = Seen::default(); + + let reported = poll(&store, &branch, "running", &seen); + + assert_eq!(reported, "error"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(seen.triggered(), Vec::::new()); + assert_eq!(seen.written(), vec![]); + } +} + +/// Every other transition is still the pollers'. +#[test] +fn a_poll_still_owns_the_other_transitions() { + use WorkspaceStatus::*; + for poll in [poll_batch, poll_single] { + for (row, blox_status, expected) in [ + (Running, "suspended", Suspended), + (Running, "stopped", Stopped), + (Running, "degraded", Error), + (Running, "running", Running), + (Starting, "suspended", Suspended), + (Starting, "failed", Error), + (Starting, "provisioning", Starting), + // Blox briefly reports a new workstation as stopped. + (Starting, "stopped", Starting), + // Resumed outside Staged: it was set up before it was suspended. + (Suspended, "running", Running), + (Stopped, "running", Running), + ] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, row.clone()); + let seen = Seen::default(); + + let reported = poll(&store, &branch, blox_status, &seen); + + assert_eq!(reported, expected.as_str(), "{row:?} + {blox_status}"); + assert_eq!(status_of(&store, &branch), Some(expected.clone())); + assert_eq!(seen.triggered(), Vec::::new()); + let changed = row != expected; + assert_eq!( + seen.written(), + vec![(branch.id.clone(), changed, expected)], + "{row:?} + {blox_status}" + ); + } + } +} + +/// A repo joining a workstation a peer already runs on is still held at +/// `Starting` by the pollers, and left to the setup that is cloning it. +#[test] +fn a_secondary_clone_is_still_held_at_starting() { + let (store, project) = store_with_project(); + let peer = starting_branch(&store, &project, "feature-one"); + set_status(&store, &peer, WorkspaceStatus::Running); + let repo = store::ProjectRepo::new(&project.id, "acme/gadgets", "feature-two", None); + store.create_project_repo(&repo).unwrap(); + let secondary = store::Branch::new_remote(&project.id, "feature-two", "main", WORKSPACE) + .with_project_repo(&repo.id); + store.create_branch(&secondary).unwrap(); + let seen = Seen::default(); + + assert_eq!(poll::is_secondary_clone_setup(&store, &secondary), Ok(true)); + assert_eq!(poll_batch(&store, &secondary, "running", &seen), "starting"); + + assert_eq!( + status_of(&store, &secondary), + Some(WorkspaceStatus::Starting) + ); + assert_eq!(seen.triggered(), Vec::::new()); + assert_eq!(seen.written(), vec![]); +} + +/// The poll's trigger, followed through: the branch a poll refused to +/// promote ends up checked out and `Running` by setup's hand. +#[tokio::test] +async fn a_polled_starting_branch_is_set_up_and_then_reported_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + let seen = Seen::default(); + + assert_eq!(poll_batch(&store, &branch, "running", &seen), "starting"); + for branch_id in seen.triggered() { + assert_eq!(branch_id, branch.id); + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + } + + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(head_branch(clone.path()), "feature-new"); + let branch = store.get_branch(&branch.id).unwrap().unwrap(); + assert_eq!(poll_batch(&store, &branch, "running", &seen), "running"); + assert_eq!(seen.triggered().len(), 1); +} + +// ── The startup pass ──────────────────────────────────────────────────────── + +/// The startup pass takes the remote branches a previous run left in +/// `Starting` and nothing else. +#[test] +fn the_startup_pass_picks_up_only_branches_left_starting() { + let (store, project) = store_with_project(); + let left_starting = starting_branch(&store, &project, "feature-starting"); + for (name, status) in [ + ("feature-running", WorkspaceStatus::Running), + ("feature-error", WorkspaceStatus::Error), + ("feature-suspended", WorkspaceStatus::Suspended), + ] { + let branch = starting_branch(&store, &project, name); + set_status(&store, &branch, status); + } + store + .create_branch(&store::Branch::new(&project.id, "feature-local", "main")) + .unwrap(); + + let pending: Vec = branches_awaiting_setup(&store) + .unwrap() + .into_iter() + .map(|branch| branch.id) + .collect(); + + assert_eq!(pending, vec![left_starting.id]); +} + +/// A run that quit between a workstation coming up and its checkout left +/// the row in `Starting` with nothing asking for setup. The next launch +/// finishes the job with no window involved. +#[tokio::test] +async fn the_startup_pass_finishes_a_setup_the_last_run_dropped() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + for pending in branches_awaiting_setup(&store).unwrap() { + set_up(&store, &host, &pending, SetupTrigger::Observed) + .await + .unwrap(); + } + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert!(branches_awaiting_setup(&store).unwrap().is_empty()); +} diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index d68ab3a70..945445439 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -60,6 +60,12 @@ class WorkspaceLifecycleController { // any is running, stopped otherwise. private readonly STARTING_POLL_MS = 3000; private readonly RUNNING_POLL_MS = 30_000; + // How long a branch may sit in `starting` with nothing here setting it up + // before it is shown as failed. A branch stays `starting` until the + // backend has checked it out on the workstation, not just until the + // workstation is up, and that setup is bounded and reports its own + // failure through `startWorkspace`. So the clock only runs for a branch + // this controller has no setup call queued or in flight for. private readonly STARTING_TIMEOUT_MS = 5 * 60 * 1000; private pollTimer: ReturnType | null = null; @@ -290,6 +296,11 @@ class WorkspaceLifecycleController { ); } + /** Whether a setup call for the branch is queued or in flight here. */ + private isSetupOwned(branchId: string): boolean { + return this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId); + } + private errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -339,11 +350,14 @@ class WorkspaceLifecycleController { return; } - // Apply 5-minute timeout to starting workspaces before polling. + // Apply 5-minute timeout to starting workspaces before polling. Not to + // one whose setup call is still out: its answer settles the branch, and + // timing out under it would show an error over a setup that then + // succeeds. const now = Date.now(); const activeBranchIds: string[] = []; for (const target of targets) { - if (target.status === 'starting') { + if (target.status === 'starting' && !this.isSetupOwned(target.branchId)) { const startedAt = this.startingTimestamps.get(target.branchId); if (startedAt && now - startedAt > this.STARTING_TIMEOUT_MS) { this.startingTimestamps.delete(target.branchId); @@ -469,7 +483,12 @@ class WorkspaceLifecycleController { } try { + // Resolves once the backend has the branch checked out on the + // workstation and marked `running`; rejects with why it is in + // `error`. A poll never promotes a `starting` branch, so this + // answer is what moves the branch on. await commands.startWorkspace(branchId); + this.handleWorkspaceStatusChange(projectId, branchId, 'running'); } catch (e) { console.error('[workspaceLifecycle] Failed to start workspace:', e); const message = this.errorMessage(e); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 85cc1adf2..0ef428de4 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -251,6 +251,81 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); + // A branch stays `starting` until the backend has checked it out on the + // workstation, which can outlast the five-minute starting timeout. While + // the setup call is out, its answer settles the branch: timing out under + // it would show an error over a setup that then succeeds. + it('does not time out a starting branch while its setup call is in flight', async () => { + vi.useFakeTimers(); + let finishSetup!: () => void; + commandsMock.startWorkspace.mockReturnValue( + new Promise((resolve) => (finishSetup = resolve)) + ); + // The poll never promotes it: only setup does. + commandsMock.pollAllWorkspaceStatuses.mockResolvedValue({ + 'branch-1': { status: 'starting', workstationId: null }, + }); + branchesByProject = new Map([ + ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], + ]); + const lifecycle = await newController(); + await kickoff(lifecycle); + expect(commandsMock.startWorkspace).toHaveBeenCalledWith('branch-1'); + + await vi.advanceTimersByTimeAsync(7 * 60 * 1000); + expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('starting'); + expect(lifecycle.getWorkspaceErrors().size).toBe(0); + + finishSetup(); + await settle(); + expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('running'); + lifecycle.stop(); + }); + + // The timeout still catches a branch stuck in `starting` that nothing here + // is setting up, so it cannot spin forever. + it('times out a starting branch it has no setup call for', async () => { + vi.useFakeTimers(); + commandsMock.pollAllWorkspaceStatuses.mockResolvedValue({ + 'branch-1': { status: 'starting', workstationId: null }, + }); + branchesByProject = new Map([ + ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], + ]); + // Setup skips a branch whose type disagrees with its project's location. + projectLocations = new Map([['project-1', 'local']]); + const lifecycle = await newController(); + await kickoff(lifecycle); + expect(commandsMock.startWorkspace).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(4 * 60 * 1000); + expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('starting'); + await vi.advanceTimersByTimeAsync(2 * 60 * 1000); + expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('error'); + lifecycle.stop(); + }); + + // A setup that fails leaves the branch in `error` in the backend and + // rejects; the branch shows the reason and is no longer polled. + it('shows the setup error when startWorkspace rejects', async () => { + vi.useFakeTimers(); + commandsMock.startWorkspace.mockRejectedValue( + new Error("Failed to set up 'acme/widgets' in workspace 'ws': could not fetch") + ); + branchesByProject = new Map([ + ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], + ]); + const lifecycle = await newController(); + await kickoff(lifecycle); + + expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('error'); + expect(lifecycle.getWorkspaceErrors().get('branch-1')).toContain('could not fetch'); + const polls = commandsMock.pollAllWorkspaceStatuses.mock.calls.length; + await vi.advanceTimersByTimeAsync(60 * 1000); + expect(commandsMock.pollAllWorkspaceStatuses.mock.calls.length).toBe(polls); + lifecycle.stop(); + }); + // The backend kicks the branch queue when it writes the `running` workspace // status, so the frontend must not issue its own drain on the transition — // the only command a `running` transition may trigger is the status poll. From 053f96ccbe6a16c8330886d2e3ca878facaa878a Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Tue, 17 Nov 2026 17:04:03 +1100 Subject: [PATCH 2/9] fix(staged): verify the clone is on the branch before Running and before an agent starts Follow-up to 0e0fdb4a, which made branch setup the only path from `Starting` to `Running`. Two holes were left: nothing read the clone's HEAD, so a `Running` row still rested on a checkout reporting success, and a branch that was never set up, resumed outside Staged and promoted from `Stopped` or `Suspended` by a poller, could still run sessions on `main`. The checkout routine's local-branch probe also read any failed exec as "branch missing". Requested: confirm the clone's checked-out branch at the end of setup and at remote session start, never start an agent on the wrong branch, never let a failed exec read as either answer, and make the local-branch probe tell "missing" from a failed exec. One cheap exec per session start, no fetch. How: - New `workspace_setup::verify`. The clone's HEAD is read with one transient exec of a script that answers on stdout: `branch `, `detached` or `norepo`. Anything else, and any failed exec, is an error and never an answer. - End of setup: after the checkout, the clone must read as on the branch before `Running` is written. A mismatch or an unreadable clone is a setup failure and the row goes to `Error`. - Session start: fresh branch sessions (immediate and queued) and resumed ones (manual, claimed and re-queued) make the same read before anything else reads the clone and before the row is claimed. - On a mismatch the session start repairs instead of failing. It runs the shared setup under the guard with a new `SetupTrigger::Repair`, then reads the clone again and starts only on a read that matches. Failing outright would end every session queued on the branch in turn with the same error, and nothing else asks for setup of a `Running` branch. During the repair the row is `Starting`, and `Error` if it fails, so the session queue's existing gate holds the rest of the queue. The session that asked fails with the expected and the actual branch and the cause. - The repair is refused when another live session works in the same clone (two branches of one repository share a clone). The start fails with both branch names rather than check out under a running agent. - A read that does not answer is retried once after 2 s, each attempt capped at 30 s, and then fails the start. It triggers no checkout and does not let the agent start. A queued start also gets the queue's own bounded pre-claim retries. - `check_out_workspace_branch` asks `for-each-ref` whether the clone has the branch. It exits 0 either way and prints the ref only when it exists, so a failed exec is now an error instead of a `checkout -B` from the base that would move an existing branch off its unpushed commits. Tests, through the fake host and the `GitRunner` injection, with no `sq` call: the probe script against real clones, setup refusing `Running` over a wrong or unreadable clone, the session start reading once, repairing a clone left on `main`, failing visibly when the repair fails, treating an unanswered read as neither answer, refusing to switch a clone a live session works in, and the failed local-branch probe leaving the branch untouched. Not in this commit: pipeline sessions (rebase, squash, push, PR) do not make this check before their git steps, and a reattach to an agent that is already running does not either. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- apps/staged/src-tauri/src/branches.rs | 36 +- apps/staged/src-tauri/src/session_commands.rs | 79 ++- apps/staged/src-tauri/src/workspace_setup.rs | 101 +++- .../src-tauri/src/workspace_setup/tests.rs | 58 ++ .../src/workspace_setup/tests/verify.rs | 543 ++++++++++++++++++ .../src-tauri/src/workspace_setup/verify.rs | 314 ++++++++++ 6 files changed, 1099 insertions(+), 32 deletions(-) create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/verify.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/verify.rs diff --git a/apps/staged/src-tauri/src/branches.rs b/apps/staged/src-tauri/src/branches.rs index 2a7f278c8..888b43c5e 100644 --- a/apps/staged/src-tauri/src/branches.rs +++ b/apps/staged/src-tauri/src/branches.rs @@ -802,12 +802,14 @@ pub(crate) async fn check_out_branch_in_workspace( let base = base_ref.to_string(); let branch = branch_name.to_string(); tauri::async_runtime::spawn_blocking(move || { - // Tracked for everything but `rev-parse`: Blox SIGKILLs a transient - // git at 60 s, which a fetch on a large repo can outlast, and a - // checkout or fast-forward far from where the tree is (or smudging LFS - // files) can outlast it too, leaving `.git/index.lock` and a half-updated worktree. + // Tracked for everything but the ref reads (`rev-parse`, and the + // `for-each-ref` that asks whether the clone has the branch): Blox + // SIGKILLs a transient git at 60 s, which a fetch on a large repo can + // outlast, and a checkout or fast-forward far from where the tree is + // (or smudging LFS files) can outlast it too, leaving + // `.git/index.lock` and a half-updated worktree. let git: GitRunner<'_> = Box::new(move |args: &[&str]| { - let mode = if args.first() == Some(&"rev-parse") { + let mode = if matches!(args.first(), Some(&"rev-parse") | Some(&"for-each-ref")) { ExecMode::Transient } else { ExecMode::Tracked @@ -862,7 +864,11 @@ pub(crate) fn check_out_workspace_branch( } let remote_branch = format!("origin/{branch_name}"); - if git(&["rev-parse", "--verify", "--quiet", &local_ref]).is_ok() { + // Whether the clone has the branch decides between checking it out as it + // is and creating it with `checkout -B`. A failed exec is neither answer: + // read as "missing", it would recreate a branch the clone does have from + // the base, moving it off its unpushed commits. + if local_branch_exists(git, branch_name)? { git(&["checkout", branch_name]) .map_err(|e| format!("could not check out branch '{branch_name}': {e}"))?; if has_remote_branch { @@ -893,6 +899,24 @@ pub(crate) fn check_out_workspace_branch( .map_err(|e| format!("could not create branch '{branch_name}': {e}")) } +/// Whether the clone `git` runs in has a local branch `branch_name`. +/// +/// Answered through stdout, the way [`GIT_DIR_PROBE_SCRIPT`] answers: +/// `for-each-ref` exits 0 whether or not the ref exists and prints it only +/// when it does, so "missing" is an empty answer from a command that ran. A +/// probe that answers through its exit status (`rev-parse --verify --quiet`) +/// can't be told apart from an exec that never reached the workstation, and +/// here that failure is an error. +pub(crate) fn local_branch_exists(git: &GitRunner<'_>, branch_name: &str) -> Result { + let local_ref = format!("refs/heads/{branch_name}"); + let refs = git(&["for-each-ref", "--format=%(refname)", &local_ref]).map_err(|e| { + format!("could not check whether the clone already has branch '{branch_name}': {e}") + })?; + // The pattern also matches refs under `/`, so only an exact line + // counts. + Ok(refs.lines().any(|line| line.trim() == local_ref)) +} + /// Kick the branch's session queue when a workspace-status write moved it to /// `Running`. /// diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index 7d95baa53..f5b38a85c 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -1892,6 +1892,7 @@ pub(crate) async fn resume_session_for_store( } let plan = prepare_resume(&store, &session, branch_id, acp_config_selection).await?; + ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; // Live from before the `running` write until the runner has registered // (see `SessionRegistry::hold`); released when this function returns. @@ -1956,6 +1957,38 @@ fn ensure_claimed_resume_still_running(store: &Store, session_id: &str) -> Resul } } +/// A resume's share of the checkout check every agent start on a remote +/// branch makes (`workspace_setup::verify`): the clone the resumed turn runs +/// in is read to be on the session's branch, and repaired when it is not. +/// A no-op for a local branch and for a session with no branch. +/// +/// Not part of `prepare_resume`, which a reattach also reads its config +/// through: a reattach joins an agent that is already running in the clone, +/// so there is no start to hold back and no checkout to run under it. +/// +/// Unlike the pre-run HEAD snapshot this does not degrade: a resume whose +/// clone can't be read fails, since the turn would otherwise run unchecked. +/// Its waits are bounded (two capped reads; a repair's execs each carry +/// their own limit), so a claimed row is not held `running` without end. +async fn ensure_resume_checkout( + app_handle: &tauri::AppHandle, + store: &Arc, + plan: &ResumePlan, + session_id: &str, +) -> Result<(), String> { + if plan.workspace_name.is_none() { + return Ok(()); + } + let Some(branch_id) = plan.event_branch_id.as_deref() else { + return Ok(()); + }; + let Some(branch) = store.get_branch(branch_id).map_err(|e| e.to_string())? else { + return Ok(()); + }; + crate::workspace_setup::verify::ensure_session_checkout(app_handle, store, &branch, session_id) + .await +} + /// Everything a resume reads before the row is claimed: the session's /// artifact and branch, the provider and config selection, the workspace /// path and the pre-run HEAD. None of it needs the row to be `running`, so @@ -2643,12 +2676,15 @@ pub(crate) async fn send_session_message_now_for_store( /// /// The row is `running` and ours from the claim on, under a liveness hold the /// stranded-session sweep respects, so nothing in the preparation may wait -/// without bound: its one wait, the pre-run HEAD read, runs under -/// [`RESUME_HEAD_READ_CAP`] inside `prepare_resume` and degrades to no -/// snapshot, so a slow checkout delays the resume by at most the cap and does -/// not fail it. A preparation or launch that does fail is recorded on the -/// row as `error`, and the hold is released when this returns either way, so -/// the branch drains. +/// without bound: the pre-run HEAD read runs under [`RESUME_HEAD_READ_CAP`] +/// inside `prepare_resume` and degrades to no snapshot, so a slow checkout +/// delays the resume by at most the cap and does not fail it. The check that +/// a remote clone is on the session's branch ([`ensure_resume_checkout`]) is +/// bounded too but does not degrade: a clone that can't be read, or can't be +/// put on the branch, fails the resume rather than run the turn somewhere +/// else. A preparation or launch that does fail is recorded on the row as +/// `error`, and the hold is released when this returns either way, so the +/// branch drains. #[allow(clippy::too_many_arguments)] pub(crate) async fn resume_claimed_session_for_store( store: Arc, @@ -2661,7 +2697,13 @@ pub(crate) async fn resume_claimed_session_for_store( image_ids: Vec, ) -> Result<(), String> { let ClaimedResume { session, _liveness } = claim; - let plan = match prepare_resume(&store, &session, None, None).await { + let prepared = match prepare_resume(&store, &session, None, None).await { + Ok(plan) => ensure_resume_checkout(&app_handle, &store, &plan, &session.id) + .await + .map(|()| plan), + Err(e) => Err(e), + }; + let plan = match prepared { Ok(plan) => plan, Err(e) => { let branch_id = store.get_branch_id_for_session(&session.id).ok().flatten(); @@ -5299,6 +5341,12 @@ async fn prepare_branch_session_start( let working_dir = target.working_dir.clone(); let is_remote = branch.workspace_name.is_some(); + // A remote branch's clone is read to be on the branch before the context + // below is built from it and an agent is started in it (a no-op for a + // local branch). See `workspace_setup::verify`. + crate::workspace_setup::verify::ensure_session_checkout(app_handle, store, branch, session_id) + .await?; + // Remote branches use ws_exec for git operations; local branches use the // worktree directly. let (branch_context, pikchr_grammar_reference, note_draft_path) = if is_remote { @@ -6847,6 +6895,20 @@ async fn start_queued_session_for_branch( session.provider.clone().or(provider) }; + // A remote branch's clone is read to be on the branch before the context + // below is built from it and an agent is started in it (a no-op for a + // local branch; see `workspace_setup::verify`). Before the claim, like + // everything else here that awaits: a read that fails leaves the row + // queued for the drain's bounded retries, and a repair that fails leaves + // the branch in `Error`, where the queue holds its rows. + crate::workspace_setup::verify::ensure_session_checkout( + &app_handle, + &store, + &branch, + &session_id, + ) + .await?; + // Resolve working directory and branch context. let (working_dir, branch_context, pikchr_grammar_reference, note_draft_path) = if is_remote { let fallback_dir = resolve_branch_repo_slug(&store, &project, &branch) @@ -7141,6 +7203,9 @@ async fn start_requeued_session_for_branch( // (`RESUME_HEAD_READ_CAP`) and degrades to no snapshot, so a slow checkout // neither holds this branch's drain open nor spends a retry. let plan = prepare_resume(&store, &session, Some(branch_id.clone()), None).await?; + // Before the claim too: a failed read spends one of the drain's retries + // and leaves the row queued. + ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; // Live from before the claim flips the row to `running` until // `start_session` has registered it (see `SessionRegistry::hold`). diff --git a/apps/staged/src-tauri/src/workspace_setup.rs b/apps/staged/src-tauri/src/workspace_setup.rs index 1337557fc..7be516531 100644 --- a/apps/staged/src-tauri/src/workspace_setup.rs +++ b/apps/staged/src-tauri/src/workspace_setup.rs @@ -4,10 +4,11 @@ //! running, and the workstation's clone is on the branch. Blox only knows //! about the first. This module owns the second, and with it the one write //! that says both hold: it is the only code that moves a branch from -//! `Starting` to `Running`, and it does so only after the checkout succeeded. -//! A failure leaves the branch in `Error` and is returned, never logged and -//! dropped — a branch marked `Running` on a workstation still on its base -//! branch has its sessions commit onto the base. +//! `Starting` to `Running`, and it does so only after the checkout succeeded +//! and the clone reads as on the branch ([`verify`]). A failure leaves the +//! branch in `Error` and is returned, never logged and dropped — a branch +//! marked `Running` on a workstation still on its base branch has its +//! sessions commit onto the base. //! //! Setup is level-triggered. Whoever sees a `Starting` branch asks for it: //! @@ -19,6 +20,10 @@ //! - the startup pass ([`resume_pending_setups`]), for branches a previous //! run left in `Starting`. //! +//! One more caller asks for a branch that is already `Running`: an agent +//! start that read the clone and found it off the branch +//! ([`verify::ensure_session_checkout`], [`SetupTrigger::Repair`]). +//! //! All of them go through one [`guard::SetupGuard`], so a second request for //! a branch joins the run in flight, and two branches never run setup in one //! workstation at once. The guard is per process: a second Staged instance @@ -28,6 +33,7 @@ pub(crate) mod guard; pub(crate) mod poll; #[cfg(test)] mod tests; +pub(crate) mod verify; use std::sync::Arc; use std::time::Duration; @@ -39,6 +45,7 @@ use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; use guard::SetupGuard; +use verify::{CloneHead, HeadRead}; /// Who asked for a branch to be set up. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -51,6 +58,12 @@ pub(crate) enum SetupTrigger { /// the startup pass. It acts only while the row still says so, so a /// trigger that arrives after setup settled the branch is a no-op. Observed, + /// An agent start that read the clone of a `Running` branch and found it + /// off the branch (see [`verify`]). The only trigger that acts on a + /// `Running` row, and only while the clone still reads wrong when its + /// turn comes. The row goes back to `Starting` for the run, so nothing + /// else starts on the branch meanwhile, and to `Error` if it fails. + Repair, } /// How long setup waits for a workstation to report running after `sq blox @@ -104,6 +117,11 @@ pub(crate) trait SetupHost: Send + Sync + 'static { /// ([`branches::check_out_workspace_branch`]). async fn check_out(&self, checkout: &Checkout) -> Result<(), String>; + /// What the clone has checked out: one cheap read, no fetch + /// ([`verify::read_clone_head`]). `Err` when the read didn't answer, + /// which says nothing about the clone. + async fn clone_head(&self, checkout: &Checkout) -> Result; + /// Called with what [`Store::update_branch_workspace_status`] reported /// for the `Running` write (see [`branches::workspace_status_written`]). fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus); @@ -111,6 +129,10 @@ pub(crate) trait SetupHost: Send + Sync + 'static { fn ready_wait(&self) -> ReadyWait { ReadyWait::DEFAULT } + + fn head_read(&self) -> HeadRead { + HeadRead::DEFAULT + } } /// The real host: `sq blox` and the workstation's own git. @@ -177,6 +199,10 @@ impl SetupHost for BloxHost { } } + async fn clone_head(&self, checkout: &Checkout) -> Result { + verify::read_clone_head(&checkout.workspace, checkout.clone_dir.as_deref()).await + } + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { branches::workspace_status_written(&self.app_handle, branch_id, changed, status); } @@ -321,12 +347,17 @@ pub(crate) async fn run_guarded( /// Whether `trigger` has anything to do for a branch in this state. A /// `Running` branch is set up already, and may have a session in its clone: -/// nothing touches it. An observed trigger acts only on `Starting`. +/// nothing touches it but a repair, which an agent start asks for after +/// reading the clone off the branch. An observed trigger acts only on +/// `Starting`. A repair acts on nothing else either: a branch that has +/// stopped, or failed, since the session read its clone is not one to start +/// a workstation for from a session start. fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { match (&branch.workspace_status, trigger) { + (Some(WorkspaceStatus::Running), SetupTrigger::Repair) => true, (Some(WorkspaceStatus::Running), _) => false, - (Some(WorkspaceStatus::Starting), SetupTrigger::Observed) => true, - (_, SetupTrigger::Observed) => false, + (Some(WorkspaceStatus::Starting), SetupTrigger::Observed | SetupTrigger::Repair) => true, + (_, SetupTrigger::Observed | SetupTrigger::Repair) => false, (_, SetupTrigger::Requested) => true, } } @@ -338,7 +369,9 @@ fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { /// already running (a resumed workstation, a repo joining a shared one, a /// retry after a restart). Both end in /// [`branches::check_out_workspace_branch`], which is idempotent and never -/// resets a branch the clone already has. +/// resets a branch the clone already has, and then in +/// [`verify::confirm_checked_out`]: `Running` is written only over a clone +/// that reads as on the branch. /// /// Every failure past the first lookups leaves the row in `Error` and is /// returned. Callers go through [`run_guarded`]. @@ -361,10 +394,29 @@ pub(crate) async fn set_up_branch( return Ok(()); } + // A repair was asked for by a session that read the clone off the + // branch. The run ahead of this one (another session's repair, a + // sibling branch's setup) may have changed that; when the clone reads + // right now there is nothing to repair, and a `Running` branch is left + // exactly as it is. A read that fails here settles nothing: the repair + // goes ahead and ends in its own read. + if trigger == SetupTrigger::Repair && branch.workspace_status == Some(WorkspaceStatus::Running) + { + if let Ok(checkout) = checkout_for(store, &branch) { + if matches!( + host.clone_head(&checkout).await, + Ok(CloneHead::Branch(name)) if name == checkout.branch_name + ) { + return Ok(()); + } + } + } + // A request for a branch that had settled somewhere else (a failed - // setup, a stale `starting` in the window that asked) puts it back in - // `Starting`, where the pollers hold it rather than promote it under - // this run. + // setup, a stale `starting` in the window that asked, a repair of a + // `Running` one) puts it back in `Starting`, where the pollers hold it + // rather than promote it under this run and the session queue holds + // whatever else would start on the branch. if branch.workspace_status != Some(WorkspaceStatus::Starting) { store .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) @@ -402,11 +454,9 @@ pub(crate) async fn set_up_branch( result } -async fn provision_and_check_out( - store: &Arc, - host: &dyn SetupHost, - branch: &store::Branch, -) -> Result<(), String> { +/// Where `branch` is checked out and what its clone should be on, from its +/// row and its project's. +pub(crate) fn checkout_for(store: &Arc, branch: &store::Branch) -> Result { let project = store .get_project(&branch.project_id) .map_err(|e| e.to_string())? @@ -415,13 +465,22 @@ async fn provision_and_check_out( .workspace_name .as_deref() .ok_or("Branch has no workspace name")?; - let checkout = Checkout { + Ok(Checkout { workspace: workspace.to_string(), clone_dir: branches::resolve_branch_clone_dir(store, branch)?, repo_slug: branches::resolve_branch_repo_slug(store, &project, branch)?, base_ref: branches::normalize_branch_ref(&branch.base_branch), branch_name: branch.branch_name.clone(), - }; + }) +} + +async fn provision_and_check_out( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, +) -> Result<(), String> { + let checkout = checkout_for(store, branch)?; + let workspace = checkout.workspace.as_str(); // Pre-flight: without it `blox ws start` can hang or fail opaquely, // leaving the frontend spinning for minutes (issue #99). @@ -445,7 +504,11 @@ async fn provision_and_check_out( start_workstation(store, host, branch, &checkout).await?; } - host.check_out(&checkout).await + host.check_out(&checkout).await?; + + // The checkout said it worked. `Running` rests on where the clone is, + // so that is read, not inferred. + verify::confirm_checked_out(host, &checkout).await } /// `sq blox ws start`, and when that fails without ruling the workstation diff --git a/apps/staged/src-tauri/src/workspace_setup/tests.rs b/apps/staged/src-tauri/src/workspace_setup/tests.rs index a454f6446..92cbf6063 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests.rs @@ -14,12 +14,17 @@ use tokio::sync::Semaphore; use super::guard::SetupGuard; use super::poll::{self, PollEffects, Reported}; +use super::verify::{parse_clone_head, CloneHead, HeadRead, CLONE_HEAD_PROBE_SCRIPT}; use super::{branches_awaiting_setup, run_guarded, Checkout, ReadyWait, SetupHost, SetupTrigger}; use crate::blox::{BloxError, WorkspaceListEntry}; use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; +/// The checks that the clone is on the branch: at the end of setup, at +/// session start, and the checkout routine's own local-branch probe. +mod verify; + const WORKSPACE: &str = "widgets-setup-test"; // ── Fixtures ──────────────────────────────────────────────────────────────── @@ -93,6 +98,33 @@ enum Info { Unreachable, } +/// What the fake workstation answers a read of the clone's HEAD with, in +/// place of what the clone is really on. +#[derive(Clone)] +enum Head { + /// The exec didn't answer. + Unanswered(&'static str), + /// The clone reads as this, whatever it is on. + Reads(CloneHead), +} + +/// What `clone` has checked out, asked the way the workstation is asked: +/// the real probe script, run here. +fn probe_clone_head(clone: &Path) -> Result { + let mut command = std::process::Command::new("sh"); + command + .arg("-c") + .arg(CLONE_HEAD_PROBE_SCRIPT) + .arg("_") + .arg(clone); + // Under a git hook `GIT_DIR` names the hook's repository, and the + // script's git would answer for that one. + crate::git::strip_git_env(&mut command); + let output = command.output().unwrap(); + assert!(output.status.success(), "{output:?}"); + parse_clone_head(&String::from_utf8_lossy(&output.stdout)) +} + #[derive(Clone, Copy)] enum Start { Succeeds, @@ -108,6 +140,9 @@ struct FakeHost { info: Mutex>, start: Start, checkout_error: Option<&'static str>, + /// Answers to successive reads of the clone's HEAD. Once they run out + /// the clone itself is read. + heads: Mutex>, /// When set, a checkout waits for a permit before touching the clone. gate: Option>, /// Run at each status lookup: what else happens while setup is out. @@ -125,6 +160,7 @@ impl FakeHost { info: Mutex::new(info.iter().copied().collect()), start, checkout_error: None, + heads: Mutex::new(VecDeque::new()), gate: None, meanwhile: None, calls: Mutex::new(Vec::new()), @@ -134,6 +170,11 @@ impl FakeHost { } } + fn with_heads(self, heads: &[Head]) -> Self { + *self.heads.lock().unwrap() = heads.iter().cloned().collect(); + self + } + fn gated(mut self) -> (Self, Arc) { let gate = Arc::new(Semaphore::new(0)); self.gate = Some(Arc::clone(&gate)); @@ -222,6 +263,16 @@ impl SetupHost for FakeHost { result } + async fn clone_head(&self, _checkout: &Checkout) -> Result { + self.calls.lock().unwrap().push("clone_head"); + let scripted = self.heads.lock().unwrap().pop_front(); + match scripted { + Some(Head::Unanswered(error)) => Err(error.to_string()), + Some(Head::Reads(head)) => Ok(head), + None => probe_clone_head(&self.clone), + } + } + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { self.written .lock() @@ -235,6 +286,13 @@ impl SetupHost for FakeHost { interval: Duration::from_millis(5), } } + + fn head_read(&self) -> HeadRead { + HeadRead { + attempt_cap: Duration::from_secs(10), + retry_after: Duration::from_millis(5), + } + } } async fn set_up( diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs new file mode 100644 index 000000000..5e65cfe9c --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs @@ -0,0 +1,543 @@ +//! The clone is read to be on the branch before `Running` is written and +//! before an agent starts, and a read or probe that didn't answer is never +//! taken for an answer. + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use super::super::guard::SetupGuard; +use super::super::verify::{parse_clone_head, verify_session_checkout, CloneHead}; +use super::super::{run_guarded, SetupHost, SetupTrigger}; +use super::{ + head_branch, origin_and_workstation_clone, probe_clone_head, set_status, set_up, + starting_branch, status_of, store_with_project, FakeHost, Head, Info, Start, +}; +use crate::branches; +use crate::store::{self, Store, WorkspaceStatus}; +use crate::test_utils::{run_git_in, TempDir}; + +/// A branch whose row says `Running`, the way setup or a poller left it. +fn running_branch(store: &Store, project: &store::Project, name: &str) -> store::Branch { + let branch = starting_branch(store, project, name); + set_status(store, &branch, WorkspaceStatus::Running) +} + +fn running_host(clone: &Path) -> FakeHost { + FakeHost::new(clone, &[Info::Status("running")], Start::Succeeds) +} + +/// A session row in flight on `branch`. +fn live_session(store: &Store, branch: &store::Branch) -> store::Session { + let session = store::Session::new_running("work", Path::new("/tmp")).with_branch(&branch.id); + store.create_session(&session).unwrap(); + session +} + +/// The session-start check, as `ensure_session_checkout` runs it. +async fn start_session_on( + store: &Arc, + host: &Arc, + branch: &store::Branch, + session_id: &str, +) -> Result<(), String> { + let guard = Arc::new(SetupGuard::default()); + let host: Arc = host.clone(); + verify_session_checkout(&guard, store, &host, branch, session_id).await +} + +fn calls(host: &FakeHost) -> Vec<&'static str> { + host.calls.lock().unwrap().clone() +} + +// ── The probe ─────────────────────────────────────────────────────────────── + +/// The script answers on stdout for every state of the clone, so no state +/// is told by an exit status an exec failure could also produce. +#[test] +fn the_head_probe_tells_a_branch_from_a_detached_head_from_no_repository() { + let (_origin, clone) = origin_and_workstation_clone(); + assert_eq!( + probe_clone_head(clone.path()), + Ok(CloneHead::Branch("main".to_string())) + ); + + run_git_in(clone.path(), &["checkout", "-b", "team/feature-x"]); + assert_eq!( + probe_clone_head(clone.path()), + Ok(CloneHead::Branch("team/feature-x".to_string())) + ); + + run_git_in(clone.path(), &["checkout", "--detach"]); + assert_eq!(probe_clone_head(clone.path()), Ok(CloneHead::Detached)); + + let empty = TempDir::new("no-clone"); + assert_eq!(probe_clone_head(empty.path()), Ok(CloneHead::NoRepo)); + assert_eq!( + probe_clone_head(&empty.path().join("never-created")), + Ok(CloneHead::NoRepo) + ); +} + +/// Only the script's own lines are answers. An empty or cut-off reply is an +/// error, so it can be neither "on the branch" nor "on another one". +#[test] +fn an_unrecognised_probe_reply_is_an_error() { + assert_eq!( + parse_clone_head("branch refs/heads/feature-x\n"), + Ok(CloneHead::Branch("feature-x".to_string())) + ); + assert_eq!(parse_clone_head("detached\n"), Ok(CloneHead::Detached)); + assert_eq!(parse_clone_head("norepo\n"), Ok(CloneHead::NoRepo)); + // HEAD on a ref that is not a local branch is not "on a branch". + assert_eq!( + parse_clone_head("branch refs/remotes/origin/main\n"), + Ok(CloneHead::Detached) + ); + for reply in [ + "", + "\n", + "branch", + "feature-x", + "fatal: not a git repository", + ] { + assert!(parse_clone_head(reply).is_err(), "{reply:?}"); + } +} + +// ── The end of setup ──────────────────────────────────────────────────────── + +/// A checkout that reports success and leaves the clone on the base branch +/// is a failed setup: `Running` is never written over it. +#[tokio::test] +async fn setup_does_not_say_running_over_a_clone_on_another_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new( + running_host(clone.path()) + .with_heads(&[Head::Reads(CloneHead::Branch("main".to_string()))]), + ); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("branch 'main'"), "{error}"); + assert!(error.contains("'feature-new'"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.written(), vec![]); +} + +/// A detached HEAD after setup is not the branch either. +#[tokio::test] +async fn setup_does_not_say_running_over_a_detached_head() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(running_host(clone.path()).with_heads(&[Head::Reads(CloneHead::Detached)])); + + let error = set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap_err(); + + assert!(error.contains("a detached HEAD"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); +} + +/// The read that confirms the checkout failing is not a confirmation: after +/// its one retry setup fails, though the checkout itself worked. +#[tokio::test] +async fn setup_fails_when_the_checked_out_branch_cannot_be_read() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(running_host(clone.path()).with_heads(&[ + Head::Unanswered("workstation disconnected"), + Head::Unanswered("sq blox command timed out after 300s"), + ])); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("Could not verify"), "{error}"); + assert!(error.contains("timed out"), "{error}"); + assert_eq!(host.count("clone_head"), 2); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.written(), vec![]); +} + +/// One read that didn't answer is read again, and a second that does +/// settles it. +#[tokio::test] +async fn one_unanswered_read_at_the_end_of_setup_is_read_again() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new( + running_host(clone.path()).with_heads(&[Head::Unanswered("workstation disconnected")]), + ); + + set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!(host.count("clone_head"), 2); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +// ── Session start ─────────────────────────────────────────────────────────── + +/// The usual start costs one read and nothing else: no fetch, no checkout, +/// no write. +#[tokio::test] +async fn a_session_on_a_checked_out_branch_starts_after_one_read() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + run_git_in(clone.path(), &["checkout", "-b", "feature-new"]); + let host = Arc::new(running_host(clone.path())); + + start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap(); + + assert_eq!(calls(&host), vec!["clone_head"]); + assert_eq!(host.written(), vec![]); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// The gap setup alone can't close: a branch that was never set up, resumed +/// outside Staged and promoted to `Running` by a poller, still on `main`. +/// The session start checks the branch out through the shared setup and +/// only then lets the agent start. The session being started is itself +/// `running` on the branch by then and does not stand in the way. +#[tokio::test] +async fn a_session_start_repairs_a_clone_left_on_the_base_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + let session = live_session(&store, &branch); + let host = Arc::new(running_host(clone.path())); + + start_session_on(&store, &host, &branch, &session.id) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(host.count("check_out"), 1); + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!( + host.written(), + vec![(branch.id.clone(), true, WorkspaceStatus::Running)] + ); +} + +/// A clone with nothing in it reads as `norepo`, a definite answer, and is +/// repaired like any other clone that is not on the branch. +#[tokio::test] +async fn a_session_start_repairs_a_missing_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + let host = Arc::new(running_host(clone.path()).with_heads(&[ + // The session's read, then the repair's look before it starts. + Head::Reads(CloneHead::NoRepo), + Head::Reads(CloneHead::NoRepo), + ])); + + start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap(); + + assert_eq!(host.count("check_out"), 1); + assert_eq!(head_branch(clone.path()), "feature-new"); +} + +/// When the repair fails the agent is not started: the session fails with +/// both branch names and the cause, and the branch stops saying `Running`, +/// so whatever is queued behind it waits instead of failing in turn. +#[tokio::test] +async fn a_failed_repair_fails_the_session_and_the_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + let mut host = running_host(clone.path()); + host.checkout_error = Some("could not fetch base branch 'main': network unreachable"); + let host = Arc::new(host); + + let error = start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap_err(); + + assert!(error.contains("branch 'main'"), "{error}"); + assert!(error.contains("'feature-new'"), "{error}"); + assert!(error.contains("network unreachable"), "{error}"); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.written(), vec![]); +} + +/// A read that doesn't answer is not "wrong branch" (no checkout runs) and +/// not "right branch" (the start fails), whatever the clone is on. It is +/// read twice, then the start fails and the branch is left as it was. +#[tokio::test] +async fn an_unanswered_read_is_neither_a_wrong_branch_nor_a_right_one() { + for clone_is_on in ["main", "feature-new"] { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + if clone_is_on != "main" { + run_git_in(clone.path(), &["checkout", "-b", clone_is_on]); + } + let host = Arc::new(running_host(clone.path()).with_heads(&[ + Head::Unanswered("workstation disconnected"), + Head::Unanswered("workstation disconnected"), + ])); + + let error = start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap_err(); + + assert!(error.contains("Could not verify"), "{error}"); + assert!(error.contains("workstation disconnected"), "{error}"); + assert_eq!(calls(&host), vec!["clone_head", "clone_head"]); + assert_eq!(head_branch(clone.path()), clone_is_on); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + } +} + +/// One blip costs a second read, not the session. +#[tokio::test] +async fn a_session_start_reads_once_more_after_an_unanswered_read() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + run_git_in(clone.path(), &["checkout", "-b", "feature-new"]); + let host = Arc::new( + running_host(clone.path()).with_heads(&[Head::Unanswered("workstation disconnected")]), + ); + + start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap(); + + assert_eq!(calls(&host), vec!["clone_head", "clone_head"]); +} + +/// Two branches of one repository share a clone. While a session of one is +/// live in it the clone is not switched under that agent: the other +/// branch's session fails, naming both. Once the clone is free the same +/// start repairs it. +#[tokio::test] +async fn a_clone_another_live_session_works_in_is_not_switched() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let sibling = running_branch(&store, &project, "feature-a"); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + let sibling_session = live_session(&store, &sibling); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + assert!(error.contains("branch 'feature-a'"), "{error}"); + assert!(error.contains("'feature-b'"), "{error}"); + assert!(error.contains("running in the same clone"), "{error}"); + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + + store + .update_session_status( + &sibling_session.id, + store::SessionStatus::Completed, + None, + None, + ) + .unwrap(); + start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-b"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A repair whose turn comes after the clone was put right (another +/// session's repair ran first) leaves the `Running` branch untouched. +#[tokio::test] +async fn a_repair_that_finds_the_clone_on_the_branch_touches_nothing() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + run_git_in(clone.path(), &["checkout", "-b", "feature-new"]); + let host = Arc::new(running_host(clone.path())); + + set_up(&store, &host, &branch, SetupTrigger::Repair) + .await + .unwrap(); + + assert_eq!(calls(&host), vec!["clone_head"]); + assert_eq!(host.written(), vec![]); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A repair does not start a workstation. For a branch that stopped after +/// the session read its clone, the repair has nothing to do and the start +/// fails on the read that follows it, naming both branches. +#[tokio::test] +async fn a_session_start_on_a_stopped_branch_fails_without_a_repair() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Stopped); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-1") + .await + .unwrap_err(); + + assert!(error.contains("branch 'main'"), "{error}"); + assert!(error.contains("'feature-new'"), "{error}"); + assert_eq!(calls(&host), vec!["clone_head", "clone_head"]); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Stopped)); +} + +/// Two sessions that both read the clone off the branch get one repair. +#[tokio::test] +async fn concurrent_session_starts_share_one_repair() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + let (host, gate) = running_host(clone.path()).gated(); + let host = Arc::new(host); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + let starts: Vec<_> = ["session-1", "session-2"] + .into_iter() + .map(|session_id| { + let (guard, store, host, branch) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&dyn_host), + branch.clone(), + ); + tokio::spawn(async move { + verify_session_checkout(&guard, &store, &host, &branch, session_id).await + }) + }) + .collect(); + + host.wait_for_checkouts(1).await; + gate.add_permits(2); + for start in starts { + assert_eq!(start.await.unwrap(), Ok(())); + } + + assert_eq!(host.count("check_out"), 1); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A repair that is the only trigger in flight is a real setup, so a +/// request that joins it gets a branch that was set up. +#[tokio::test] +async fn a_repair_runs_the_shared_setup_under_the_guard() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-new"); + let host = Arc::new(running_host(clone.path())); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + run_guarded(&guard, &store, &dyn_host, &branch.id, SetupTrigger::Repair) + .await + .unwrap(); + + assert_eq!(host.count("check_auth"), 1); + assert_eq!(host.count("check_out"), 1); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert!(!guard.is_in_flight(&branch.id)); +} + +// ── The checkout routine's local-branch probe ─────────────────────────────── + +/// A clone on `main` that also has `feature-x`, one commit ahead of origin. +fn clone_with_an_unpushed_feature_branch() -> (TempDir, TempDir, String) { + let (origin, clone) = origin_and_workstation_clone(); + run_git_in(clone.path(), &["checkout", "-b", "feature-x"]); + std::fs::write(clone.path().join("file.txt"), "base\nunpushed\n").unwrap(); + run_git_in(clone.path(), &["commit", "-am", "unpushed"]); + let unpushed = run_git_in(clone.path(), &["rev-parse", "HEAD"]); + run_git_in(clone.path(), &["checkout", "main"]); + (origin, clone, unpushed) +} + +/// The probe that asks whether the clone has the branch used to answer by +/// exit status, so an exec that never ran read as "missing" and the branch +/// was recreated from the base with `checkout -B`, off its unpushed commit. +/// A failed probe is now an error and nothing is checked out. +#[test] +fn a_failed_local_branch_probe_is_not_read_as_a_missing_branch() { + let (_origin, clone, unpushed) = clone_with_an_unpushed_feature_branch(); + let real = branches::git_runner(clone.path(), None, None); + let ran = Mutex::new(Vec::::new()); + let git: branches::GitRunner<'_> = Box::new(|args: &[&str]| { + ran.lock().unwrap().push(args.join(" ")); + if matches!(args.first(), Some(&"for-each-ref") | Some(&"rev-parse")) { + return Err("sq blox command failed: workstation disconnected".to_string()); + } + real(args) + }); + + let error = branches::check_out_workspace_branch(&git, "main", "feature-x").unwrap_err(); + + assert!(error.contains("already has branch 'feature-x'"), "{error}"); + assert!(error.contains("workstation disconnected"), "{error}"); + let ran = ran.lock().unwrap(); + assert!( + !ran.iter().any(|command| command.starts_with("checkout")), + "{ran:?}" + ); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!( + run_git_in(clone.path(), &["rev-parse", "feature-x"]), + unpushed + ); +} + +/// The probe that ran: an existing branch is checked out as it is, unpushed +/// commit and all, and a missing one is created. +#[test] +fn the_local_branch_probe_answers_through_its_output() { + let (_origin, clone, unpushed) = clone_with_an_unpushed_feature_branch(); + let git = branches::git_runner(clone.path(), None, None); + + assert_eq!(branches::local_branch_exists(&git, "feature-x"), Ok(true)); + assert_eq!(branches::local_branch_exists(&git, "feature-y"), Ok(false)); + // A prefix of a branch name is not that branch. + run_git_in(clone.path(), &["branch", "team/feature-z"]); + assert_eq!(branches::local_branch_exists(&git, "team"), Ok(false)); + assert_eq!( + branches::local_branch_exists(&git, "team/feature-z"), + Ok(true) + ); + + branches::check_out_workspace_branch(&git, "main", "feature-x").unwrap(); + assert_eq!(head_branch(clone.path()), "feature-x"); + assert_eq!(run_git_in(clone.path(), &["rev-parse", "HEAD"]), unpushed); + + branches::check_out_workspace_branch(&git, "main", "feature-y").unwrap(); + assert_eq!(head_branch(clone.path()), "feature-y"); + assert_eq!( + run_git_in(clone.path(), &["rev-parse", "HEAD"]), + run_git_in(clone.path(), &["rev-parse", "origin/main"]) + ); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/verify.rs b/apps/staged/src-tauri/src/workspace_setup/verify.rs new file mode 100644 index 000000000..340c7da59 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/verify.rs @@ -0,0 +1,314 @@ +//! Whether a workstation's clone is on the branch it belongs to. +//! +//! `Running` says two things about a remote branch: its workstation takes +//! commands, and the clone there is on the branch. Blox vouches for the +//! first. The second is read here, off the clone itself, at the two points +//! that rely on it: +//! +//! - at the end of setup, before `Running` is written +//! ([`confirm_checked_out`]); +//! - at every branch session start on a remote branch, a fresh session or +//! a resumed one, before the agent is launched +//! ([`ensure_session_checkout`]). A branch can be `Running` over a clone +//! setup never put there: a poller promotes a `Stopped` or `Suspended` row +//! when the workstation is resumed outside Staged, two branches of one +//! repo share a clone, someone switches branches in a terminal. +//! +//! The read is one transient exec and no fetch. It answers through stdout +//! ([`CLONE_HEAD_PROBE_SCRIPT`]), so an exec that failed is not an answer: +//! it is never "wrong branch" (which would run a checkout the clone may not +//! need) and never "right branch" (which would start the agent unchecked). +//! [`read_head`] tries once more after a short pause and then fails. +//! +//! A session start that finds the clone somewhere else repairs it rather +//! than failing: it runs the shared setup under the guard +//! ([`SetupTrigger::Repair`]) and starts once the clone reads right. Failing +//! instead would end every session queued on the branch one after another +//! with the same error and leave nothing that puts the clone right, since +//! nothing asks for setup of a `Running` branch. The repair moves the row to +//! `Starting` for as long as it runs and to `Error` when it fails, so the +//! session queue's own gate holds whatever else is queued until the branch +//! is usable. The one case that fails without a repair is a clone another +//! live session is working in: a checkout there would move that session's +//! work onto this branch. + +use std::sync::Arc; +use std::time::Duration; + +use crate::branches; +use crate::store::{self, Store}; + +use super::guard::SetupGuard; +use super::{checkout_for, run_guarded, BloxHost, Checkout, SetupHost, SetupTrigger}; + +/// What a clone has checked out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CloneHead { + /// On this local branch. + Branch(String), + /// A repository whose HEAD is not on a local branch. + Detached, + /// No repository where the clone should be. + NoRepo, +} + +impl std::fmt::Display for CloneHead { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CloneHead::Branch(name) => write!(f, "branch '{name}'"), + CloneHead::Detached => f.write_str("a detached HEAD"), + CloneHead::NoRepo => f.write_str("no repository"), + } + } +} + +/// Prints what the repository at `$1` has checked out: `branch `, +/// `detached`, or `norepo`. +/// +/// Every outcome is a line on stdout from a script that exits 0. Asked as a +/// bare `git symbolic-ref`, a detached HEAD and a missing clone would be +/// non-zero exits, which over `ws exec` look the same as a workstation that +/// never ran the command. +pub(crate) const CLONE_HEAD_PROBE_SCRIPT: &str = r#"if ref=$(git -C "$1" symbolic-ref -q HEAD 2>/dev/null); then printf 'branch %s\n' "$ref"; elif git -C "$1" rev-parse --git-dir >/dev/null 2>&1; then echo detached; else echo norepo; fi"#; + +/// Read [`CLONE_HEAD_PROBE_SCRIPT`]'s answer. Anything the script doesn't +/// print (an empty reply, a truncated one) is an error, not a guess. +pub(crate) fn parse_clone_head(output: &str) -> Result { + let answer = output.trim(); + if answer == "detached" { + return Ok(CloneHead::Detached); + } + if answer == "norepo" { + return Ok(CloneHead::NoRepo); + } + match answer.strip_prefix("branch ") { + Some(head_ref) => Ok(match head_ref.trim().strip_prefix("refs/heads/") { + Some(name) if !name.is_empty() => CloneHead::Branch(name.to_string()), + // HEAD names a ref that is not a local branch. + _ => CloneHead::Detached, + }), + None => Err(format!( + "unexpected answer from the checked-out-branch probe: {answer:?}" + )), + } +} + +/// What the clone at `clone_dir` on `workspace` has checked out: one +/// transient exec. `clone_dir` is the `home:` form, or `None` for a +/// branch with no project repo, whose git runs in the workstation's default +/// directory. +pub(crate) async fn read_clone_head( + workspace: &str, + clone_dir: Option<&str>, +) -> Result { + let dir = match clone_dir { + Some(dir) => branches::resolve_workspace_repo_path(workspace, dir) + .map_err(|e| format!("could not resolve workspace repo path '{dir}': {e}"))?, + None => ".".to_string(), + }; + let workspace = workspace.to_string(); + let output = branches::run_blox_blocking(move || { + branches::run_workspace_shell(&workspace, CLONE_HEAD_PROBE_SCRIPT, &[&dir]) + }) + .await + .map_err(|e| e.to_string())?; + parse_clone_head(&output) +} + +/// How [`read_head`] treats a read that doesn't answer. +#[derive(Debug, Clone, Copy)] +pub(crate) struct HeadRead { + /// How long one read may take. The exec itself gives up much later + /// (`blox-cli`'s 300 s); a session start doesn't wait that long for one + /// `symbolic-ref`. + pub(crate) attempt_cap: Duration, + /// The pause before the second, and last, read. + pub(crate) retry_after: Duration, +} + +impl HeadRead { + pub(crate) const DEFAULT: HeadRead = HeadRead { + attempt_cap: Duration::from_secs(30), + retry_after: Duration::from_secs(2), + }; +} + +/// Read what `checkout`'s clone is on, trying a second time when the first +/// read fails or outlives its cap. +/// +/// One retry, then an error. A transient exec fails at once on a workstation +/// that has just dropped its connection and usually works a moment later, so +/// one more read saves a session from a blip. Past that the failure is +/// returned: the caller must not treat it as either answer, and a queued +/// start already gets the session queue's own bounded retries on top. +pub(crate) async fn read_head( + host: &dyn SetupHost, + checkout: &Checkout, +) -> Result { + let policy = host.head_read(); + let first = match read_head_once(host, checkout, policy.attempt_cap).await { + Ok(head) => return Ok(head), + Err(e) => e, + }; + log::warn!( + "[workspace_setup] could not read HEAD of '{}' in workspace '{}', reading once more: {first}", + checkout.branch_name, + checkout.workspace + ); + tokio::time::sleep(policy.retry_after).await; + read_head_once(host, checkout, policy.attempt_cap).await +} + +async fn read_head_once( + host: &dyn SetupHost, + checkout: &Checkout, + cap: Duration, +) -> Result { + // A read that outlives the cap is dropped. It only reads, so there is + // nothing to undo; the exec finishes on its blocking thread and its + // answer is discarded. + match tokio::time::timeout(cap, host.clone_head(checkout)).await { + Ok(result) => result, + Err(_) => Err(format!("no answer within {}s", cap.as_secs())), + } +} + +fn is_on_branch(head: &CloneHead, checkout: &Checkout) -> bool { + matches!(head, CloneHead::Branch(name) if *name == checkout.branch_name) +} + +fn unverified(checkout: &Checkout, error: &str) -> String { + format!( + "Could not verify that branch '{}' is checked out in workspace '{}': {error}", + checkout.branch_name, checkout.workspace + ) +} + +/// The last step of setup: the clone must read as on the branch before the +/// row may say `Running`. A checkout that reported success and left the +/// clone elsewhere is a setup failure, and so is a read that never answers. +pub(crate) async fn confirm_checked_out( + host: &dyn SetupHost, + checkout: &Checkout, +) -> Result<(), String> { + match read_head(host, checkout).await { + Ok(head) if is_on_branch(&head, checkout) => Ok(()), + Ok(head) => Err(format!( + "Workspace '{}' has {head} checked out after setup, where branch '{}' was expected", + checkout.workspace, checkout.branch_name + )), + Err(e) => Err(unverified(checkout, &e)), + } +} + +/// Make sure `branch`'s clone is on the branch before `session_id`'s agent +/// is launched in it. A no-op for a local branch. +/// +/// Call it before anything else reads the clone for the session (the branch +/// context in its prompt, its pre-run HEAD) and before the row is claimed: +/// it can wait on a repair. +pub(crate) async fn ensure_session_checkout( + app_handle: &tauri::AppHandle, + store: &Arc, + branch: &store::Branch, + session_id: &str, +) -> Result<(), String> { + if branch.branch_type != store::BranchType::Remote || branch.workspace_name.is_none() { + return Ok(()); + } + let host: Arc = Arc::new(BloxHost { + app_handle: app_handle.clone(), + }); + verify_session_checkout(SetupGuard::global(), store, &host, branch, session_id).await +} + +/// [`ensure_session_checkout`] for a remote branch, against any host. +/// +/// Returns `Ok` only after a read that says the clone is on the branch: +/// the first one, or the one after a repair. +pub(crate) async fn verify_session_checkout( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, + session_id: &str, +) -> Result<(), String> { + let checkout = checkout_for(store, branch)?; + let found = match read_head(host.as_ref(), &checkout).await { + Ok(head) if is_on_branch(&head, &checkout) => return Ok(()), + Ok(head) => head, + Err(e) => return Err(unverified(&checkout, &e)), + }; + let (workspace, expected) = (&checkout.workspace, &checkout.branch_name); + log::warn!( + "[workspace_setup] session {session_id}: workspace '{workspace}' has {found} checked \ + out, where branch '{expected}' was expected; checking the branch out before the agent \ + starts" + ); + + if let Some(busy) = live_session_in_clone(store, branch, &checkout, session_id)? { + return Err(format!( + "Workspace '{workspace}' has {found} checked out, where this session's branch \ + '{expected}' was expected. It was not switched because a session of branch \ + '{busy}' is running in the same clone; start this session again once that one \ + has finished." + )); + } + + run_guarded(guard, store, host, &branch.id, SetupTrigger::Repair) + .await + .map_err(|e| { + format!( + "Workspace '{workspace}' had {found} checked out, where this session's branch \ + '{expected}' was expected, and checking the branch out failed: {e}" + ) + })?; + + // The repair's own word is not taken for it: a run this one joined, or a + // row that had left `Running` and so had nothing to repair, says nothing + // about where the clone is now. + match read_head(host.as_ref(), &checkout).await { + Ok(head) if is_on_branch(&head, &checkout) => Ok(()), + Ok(head) => Err(format!( + "Workspace '{workspace}' has {head} checked out, where this session's branch \ + '{expected}' was expected" + )), + Err(e) => Err(unverified(&checkout, &e)), + } +} + +/// The branch whose live session works in `checkout`'s clone, if there is +/// one besides `session_id`: a session that is `running`, or `disconnected` +/// with its remote turn possibly still going. +/// +/// A project's remote branches share one workstation, and those of one +/// repository (the same repo added under two subpaths) share one clone +/// there. A checkout under a live agent would move the commits it goes on to +/// make onto another branch. +fn live_session_in_clone( + store: &Arc, + branch: &store::Branch, + checkout: &Checkout, + session_id: &str, +) -> Result, String> { + let branches_in_project = store + .list_branches_for_project(&branch.project_id) + .map_err(|e| e.to_string())?; + for other in branches_in_project { + if other.workspace_name.as_deref() != Some(checkout.workspace.as_str()) { + continue; + } + if other.id != branch.id + && branches::resolve_branch_clone_dir(store, &other)? != checkout.clone_dir + { + continue; + } + let live = store + .get_busy_sessions_for_branch(&other.id) + .map_err(|e| e.to_string())?; + if live.iter().any(|session| session.id != session_id) { + return Ok(Some(other.branch_name)); + } + } + Ok(None) +} From 92ff5c9ae3e8e0c54b2b104e8bf50717265586e1 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Tue, 17 Nov 2026 18:10:54 +1100 Subject: [PATCH 3/9] fix(staged): give Error a way out and send every Running write through a verified checkout Applies the review of 0e0fdb4a and ee35d45a. Requested: a retry path out of `Error`, no unverified write of `Running`, a checkout that never loses or moves work, an atomic session-start check scoped to the clone, a `start_workspace` whose answer means what it says, the same check for pipelines, and tests for the wiring. Error has a way out - The remote error view has a Retry button, which asks for setup again. - Every write of `Error` sets a retry timer in the backend, backed off from one minute to thirty. It runs setup only when Blox reports the workstation running and never starts or resumes one, so a branch with no window open recovers too. The startup pass arms it for rows a previous run left in `Error`. - The reason for a failure is kept in memory and emitted as a `workspace-setup-error` event, with `get_workspace_setup_errors` for a window that missed it. It covers runs no frontend call was joined to: the startup pass, poll-triggered runs, retries, a failed resume and a poll that itself moves a row to `Error`. No database field was added. Running is written only over a verified checkout - The pollers no longer promote a `Stopped`, `Suspended` or statusless row whose workstation is running. They write `Starting` and trigger setup, which also closes the two-poll `Starting`, `Suspended`, `Running` route. - A project transfer that resumed a workspace marks `Running` only the rows its own inspection read as on their branch. - Commit, rebase, squash, push, pull request and pull pipelines read the clone before their git steps, as agent starts do. The checkout neither loses nor moves work - A clone on the base branch with commits that are on no other branch and not on origin, for a branch that exists nowhere, has the branch created from HEAD. The commits come along and local base is not reset or moved. - Any other switch that would leave commits behind, carry uncommitted changes onto another branch or orphan a detached HEAD is refused with a message naming the branches and the commits. - A refused repair of a row that was `Running` puts it back to `Running` and fails only the session that asked. Session starts and checkouts do not interleave - A per-clone gate is held by setup over its checkout and by an agent or pipeline start from its read of the clone until its row is live. - The live-session scan covers every branch in the clone on that workstation, across projects. - A `disconnected` session is handed to the reconcile service at once and waited on for a bounded time, so it stops blocking once settled. A caller gets the answer to what it asked - `start_workspace` returns the row's resulting status. A caller that joined another trigger's run which did nothing runs its own. - The call returns `starting` while the run waits for a workstation to boot or for its turn on one, so the bounded wait after a failed `ws start` no longer holds a frontend setup slot. The frontend leaves such a branch to the backend and applies no timeout of its own. Tests - Source-text wiring tests fail when a trigger, an effect or a session-start or pipeline check call is deleted, and pin that nothing outside setup writes a literal `Running`. - Behaviour tests cover the retry schedule, the stranded-commit repair and each refusal, the clone gate, disconnected sessions, cross-project scope, joined callers and the early return. - `a_poll_still_owns_the_other_transitions` no longer assumes a suspended row was set up before. `workspace_setup.rs` is split into `host`, `clone`, `workstation`, `checkout`, `retry` and `failures` submodules. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- apps/staged/scripts/board-perf/mock-tauri.js | 1 + apps/staged/src-tauri/src/branches.rs | 238 +++--- apps/staged/src-tauri/src/lib.rs | 1 + .../src/project_transfer/orchestrator.rs | 42 +- .../project_transfer/orchestrator_tests.rs | 31 + .../src/project_transfer/preflight.rs | 16 + apps/staged/src-tauri/src/prs/commit.rs | 12 +- apps/staged/src-tauri/src/prs/context.rs | 26 + apps/staged/src-tauri/src/prs/create_pr.rs | 4 +- apps/staged/src-tauri/src/prs/git_pipeline.rs | 6 +- apps/staged/src-tauri/src/prs/pull.rs | 47 +- apps/staged/src-tauri/src/prs/push.rs | 7 +- apps/staged/src-tauri/src/prs/tests/pull.rs | 1 + apps/staged/src-tauri/src/session_commands.rs | 44 +- apps/staged/src-tauri/src/web_server.rs | 18 +- .../src/web_server/blocking_guard_tests.rs | 2 + apps/staged/src-tauri/src/workspace_setup.rs | 779 +++++++++--------- .../src-tauri/src/workspace_setup/checkout.rs | 420 ++++++++++ .../src-tauri/src/workspace_setup/clone.rs | 245 ++++++ .../src-tauri/src/workspace_setup/failures.rs | 71 ++ .../src-tauri/src/workspace_setup/guard.rs | 244 +++++- .../src-tauri/src/workspace_setup/host.rs | 254 ++++++ .../src-tauri/src/workspace_setup/poll.rs | 85 +- .../src-tauri/src/workspace_setup/retry.rs | 109 +++ .../src-tauri/src/workspace_setup/tests.rs | 443 ++++------ .../src/workspace_setup/tests/checkout.rs | 320 +++++++ .../src/workspace_setup/tests/poll.rs | 360 ++++++++ .../src/workspace_setup/tests/requests.rs | 231 ++++++ .../src/workspace_setup/tests/retry.rs | 331 ++++++++ .../src/workspace_setup/tests/sessions.rs | 338 ++++++++ .../src/workspace_setup/tests/verify.rs | 69 +- .../src/workspace_setup/tests/wiring.rs | 493 +++++++++++ .../src-tauri/src/workspace_setup/verify.rs | 184 ++--- .../src/workspace_setup/workstation.rs | 124 +++ apps/staged/src/lib/commands.ts | 20 +- .../lib/features/branches/BranchCard.svelte | 3 + ...emoteWorkspaceStatusView.component.test.ts | 49 ++ .../branches/RemoteWorkspaceStatusView.svelte | 8 +- .../lib/features/projects/ProjectHome.svelte | 1 + .../features/projects/ProjectSection.svelte | 3 + .../projects/workspaceLifecycle.svelte.ts | 245 ++++-- .../projects/workspaceLifecycle.test.ts | 177 +++- 42 files changed, 5064 insertions(+), 1038 deletions(-) create mode 100644 apps/staged/src-tauri/src/workspace_setup/checkout.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/clone.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/failures.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/host.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/retry.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/poll.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/requests.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/retry.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/sessions.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs create mode 100644 apps/staged/src-tauri/src/workspace_setup/workstation.rs create mode 100644 apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.component.test.ts diff --git a/apps/staged/scripts/board-perf/mock-tauri.js b/apps/staged/scripts/board-perf/mock-tauri.js index 5bb2c594a..434e1e0da 100644 --- a/apps/staged/scripts/board-perf/mock-tauri.js +++ b/apps/staged/scripts/board-perf/mock-tauri.js @@ -105,6 +105,7 @@ get_blox_env: () => null, get_interrupted_sessions: () => null, get_session_reconcile_states: () => [], + get_workspace_setup_errors: () => ({}), list_pending_control_authorizations: () => [], list_pending_project_opens: () => [], set_foreground_project: () => null, diff --git a/apps/staged/src-tauri/src/branches.rs b/apps/staged/src-tauri/src/branches.rs index 888b43c5e..2354da904 100644 --- a/apps/staged/src-tauri/src/branches.rs +++ b/apps/staged/src-tauri/src/branches.rs @@ -748,11 +748,14 @@ pub(crate) async fn clone_repo_into_workspace( ws_name: &str, clone_dir: &str, repo_slug: &str, - base_ref: &str, - branch_name: &str, -) -> Result<(), String> { - let repo_path = resolve_workspace_repo_path(ws_name, clone_dir) - .map_err(|e| format!("Failed to resolve workspace repo path '{clone_dir}': {e}"))?; + checkout: &BranchCheckout<'_>, +) -> Result<(), CheckoutError> { + let failed = CheckoutError::Failed; + let repo_path = resolve_workspace_repo_path(ws_name, clone_dir).map_err(|e| { + failed(format!( + "Failed to resolve workspace repo path '{clone_dir}': {e}" + )) + })?; // Clone only if the repo directory doesn't already exist. let probe = ws_exec_async( @@ -762,7 +765,9 @@ pub(crate) async fn clone_repo_into_workspace( ) .await; let needs_clone = needs_clone(probe).map_err(|e| { - format!("Failed to verify repo path '{clone_dir}' in workspace '{ws_name}': {e}") + failed(format!( + "Failed to verify repo path '{clone_dir}' in workspace '{ws_name}': {e}" + )) })?; if needs_clone { let repo_url = format!("https://github.com/{repo_slug}.git"); @@ -774,7 +779,11 @@ pub(crate) async fn clone_repo_into_workspace( &["sh", "-c", GUARDED_CLONE_SCRIPT, "_", &repo_url, &repo_path], ) .await - .map_err(|e| format!("Failed to clone '{repo_slug}' into workspace '{ws_name}': {e}"))?; + .map_err(|e| { + failed(format!( + "Failed to clone '{repo_slug}' into workspace '{ws_name}': {e}" + )) + })?; } // Apply staged-managed git config. Idempotent — also serves as the @@ -782,9 +791,13 @@ pub(crate) async fn clone_repo_into_workspace( // workstations, so we redo the cheap config check on every visit). crate::git::config_apply::apply_to_blox_clone(ws_name, &repo_path).await; - check_out_branch_in_workspace(ws_name, Some(clone_dir), base_ref, branch_name) + check_out_branch_in_workspace(ws_name, Some(clone_dir), checkout) .await - .map_err(|e| format!("Failed to set up '{repo_slug}' in workspace '{ws_name}': {e}")) + .map_err(|e| { + e.map_message(|m| { + format!("Failed to set up '{repo_slug}' in workspace '{ws_name}': {m}") + }) + }) } /// [`check_out_workspace_branch`] in a workstation's clone, on the blocking @@ -794,22 +807,29 @@ pub(crate) async fn clone_repo_into_workspace( pub(crate) async fn check_out_branch_in_workspace( ws_name: &str, remote_dir: Option<&str>, - base_ref: &str, - branch_name: &str, -) -> Result<(), String> { + checkout: &BranchCheckout<'_>, +) -> Result<(), CheckoutError> { let workspace = ws_name.to_string(); let remote_dir = remote_dir.map(str::to_string); - let base = base_ref.to_string(); - let branch = branch_name.to_string(); + let base = checkout.base_ref.to_string(); + let branch = checkout.branch_name.to_string(); + let base_is_worked_on = checkout.base_is_worked_on; tauri::async_runtime::spawn_blocking(move || { - // Tracked for everything but the ref reads (`rev-parse`, and the - // `for-each-ref` that asks whether the clone has the branch): Blox - // SIGKILLs a transient git at 60 s, which a fetch on a large repo can - // outlast, and a checkout or fast-forward far from where the tree is - // (or smudging LFS files) can outlast it too, leaving + // Tracked for everything but the reads (`rev-parse`, the + // `for-each-ref` that asks whether the clone has the branch, and the + // `log` and `status` that ask what a switch would leave behind): + // Blox SIGKILLs a transient git at 60 s, which a fetch on a large + // repo can outlast, and a checkout or fast-forward far from where + // the tree is (or smudging LFS files) can outlast it too, leaving // `.git/index.lock` and a half-updated worktree. let git: GitRunner<'_> = Box::new(move |args: &[&str]| { - let mode = if matches!(args.first(), Some(&"rev-parse") | Some(&"for-each-ref")) { + let mode = if matches!( + args.first(), + Some(&"rev-parse") + | Some(&"for-each-ref") + | Some(&"log") + | Some(&"--no-optional-locks") + ) { ExecMode::Transient } else { ExecMode::Tracked @@ -817,105 +837,22 @@ pub(crate) async fn check_out_branch_in_workspace( run_workspace_git_with_mode(&workspace, remote_dir.as_deref(), mode, args) .map_err(|e| e.to_string()) }); - check_out_workspace_branch(&git, &base, &branch) + check_out_workspace_branch( + &git, + &BranchCheckout { + base_ref: &base, + branch_name: &branch, + base_is_worked_on, + }, + ) }) .await - .map_err(|e| format!("Branch checkout task failed: {e}"))? + .map_err(|e| CheckoutError::Failed(format!("Branch checkout task failed: {e}")))? } -/// Fetch `base_ref` into the clone `git` runs in, and put the clone on -/// `branch_name`. -/// -/// A branch the clone already has is checked out as it is, never recreated. -/// Every remote branch of a repo shares one clone, and this runs again -/// whenever any of them re-enters `Starting` (e.g. after `resume_workspace`), -/// so `checkout -B` would reset a branch onto origin and drop its unpushed -/// commits, whichever branch HEAD happened to be on. When origin has the -/// branch too, it is then fast-forwarded if it can be: a branch that is only -/// behind picks up what was pushed, and one that has diverged keeps its local -/// commits. Only a branch the clone doesn't have yet is created, from origin's -/// copy or from the base. -pub(crate) fn check_out_workspace_branch( - git: &GitRunner<'_>, - base_ref: &str, - branch_name: &str, -) -> Result<(), String> { - git(&["fetch", "origin", base_ref]) - .map_err(|e| format!("could not fetch base branch '{base_ref}': {e}"))?; - - // If the branch already exists on the remote (e.g. from an existing PR), - // fetch it so we pick up its commits: a new branch starts from there and - // an existing one fast-forwards to it. Otherwise a new branch falls back - // to starting from the base branch. - // - // Whether it exists is read from `ls-remote`'s output, not from a fetch - // failing: a fetch that never reached origin (a dropped exec, a network - // blip) would otherwise read as "not on origin" and branch from the base, - // leaving the PR's commits behind. Any failure here is a failure. - let local_ref = format!("refs/heads/{branch_name}"); - let has_remote_branch = branch_name != base_ref - && !git(&["ls-remote", "--heads", "origin", &local_ref]) - .map_err(|e| format!("could not check whether '{branch_name}' is on origin: {e}"))? - .trim() - .is_empty(); - if has_remote_branch { - git(&["fetch", "origin", branch_name]) - .map_err(|e| format!("could not fetch branch '{branch_name}': {e}"))?; - } - let remote_branch = format!("origin/{branch_name}"); - - // Whether the clone has the branch decides between checking it out as it - // is and creating it with `checkout -B`. A failed exec is neither answer: - // read as "missing", it would recreate a branch the clone does have from - // the base, moving it off its unpushed commits. - if local_branch_exists(git, branch_name)? { - git(&["checkout", branch_name]) - .map_err(|e| format!("could not check out branch '{branch_name}': {e}"))?; - if has_remote_branch { - // Fails when the branch has diverged from origin (or local - // changes are in the way). The branch then stays as it is, local - // commits and all, which is the point: never reset it. - let _ = git(&["merge", "--ff-only", &remote_branch]); - } - return Ok(()); - } - - // `checkout -B origin/` makes the checkout track the - // branch it pushes to. A fresh branch must not track `origin/` - // instead — git's upstream is read back as the push target. - let checkout = if has_remote_branch { - git(&["checkout", "-B", branch_name, &remote_branch]) - } else { - git(&[ - "checkout", - "--no-track", - "-B", - branch_name, - &format!("origin/{base_ref}"), - ]) - }; - checkout - .map(|_| ()) - .map_err(|e| format!("could not create branch '{branch_name}': {e}")) -} - -/// Whether the clone `git` runs in has a local branch `branch_name`. -/// -/// Answered through stdout, the way [`GIT_DIR_PROBE_SCRIPT`] answers: -/// `for-each-ref` exits 0 whether or not the ref exists and prints it only -/// when it does, so "missing" is an empty answer from a command that ran. A -/// probe that answers through its exit status (`rev-parse --verify --quiet`) -/// can't be told apart from an exec that never reached the workstation, and -/// here that failure is an error. -pub(crate) fn local_branch_exists(git: &GitRunner<'_>, branch_name: &str) -> Result { - let local_ref = format!("refs/heads/{branch_name}"); - let refs = git(&["for-each-ref", "--format=%(refname)", &local_ref]).map_err(|e| { - format!("could not check whether the clone already has branch '{branch_name}': {e}") - })?; - // The pattern also matches refs under `/`, so only an exact line - // counts. - Ok(refs.lines().any(|line| line.trim() == local_ref)) -} +pub(crate) use crate::workspace_setup::checkout::{ + check_out_workspace_branch, BranchCheckout, CheckoutError, +}; /// Kick the branch's session queue when a workspace-status write moved it to /// `Running`. @@ -2439,24 +2376,23 @@ pub(crate) async fn create_remote_branch_impl( /// dialog immediately and show the card in its "Provisioning…" state while /// this runs in the background. /// -/// Returns once the branch is checked out on the workstation and marked -/// `Running`, or with the error that left it in `Error`: the shared routine -/// in [`crate::workspace_setup`], which a concurrent call for the same -/// branch joins. +/// Also the error view's Retry: a branch in `Error` is set up again. +/// +/// Returns the status the branch's row holds, so the caller does not infer +/// one from the call having resolved: `running` once the branch is checked +/// out on the workstation, or `starting` when setup is still waiting for the +/// workstation to boot, which it goes on doing after this returns. An error +/// is the failure that left the branch in `Error`. The shared routine in +/// [`crate::workspace_setup`], which a concurrent call for the same branch +/// joins. #[tauri::command(rename_all = "camelCase")] pub async fn start_workspace( store: tauri::State<'_, Mutex>>>, app_handle: AppHandle, branch_id: String, -) -> Result<(), String> { +) -> Result { let store = get_store(&store)?; - crate::workspace_setup::ensure_branch_set_up( - &app_handle, - &store, - &branch_id, - crate::workspace_setup::SetupTrigger::Requested, - ) - .await + crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id).await } /// Return the value of the `BLOX_ENV` environment variable, if set. @@ -2680,6 +2616,12 @@ pub(crate) async fn resume_workspace_impl( &store::WorkspaceStatus::Error, ) .ok(); + // `Error` with its reason on show and a retry timer, which sets the + // branches up once the workstation does come back. + let reason = format!("Could not resume workspace '{workspace_name}': {e}"); + for branch_id in &branch_ids { + crate::workspace_setup::branch_failed(app_handle, store, branch_id, &reason); + } return Err(e.to_string()); } @@ -5893,8 +5835,11 @@ mod upstream_tracking_tests { fn an_existing_clone_already_on_the_branch_is_not_reset() { let (_origin, clone, unpushed) = clone_on_feature_x_with_an_unpushed_commit(); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-x") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-x"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), @@ -5915,8 +5860,11 @@ mod upstream_tracking_tests { &["checkout", "-b", "feature-y", "origin/main"], ); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-x") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-x"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), @@ -5949,8 +5897,11 @@ mod upstream_tracking_tests { git(clone.path(), &["branch", "feature-x", "origin/feature-x"]); let pushed = push_to_origin_feature_x(&origin); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-x") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-x"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), @@ -5972,8 +5923,11 @@ mod upstream_tracking_tests { git(clone.path(), &["checkout", "main"]); let pushed = push_to_origin_feature_x(&origin); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-x") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-x"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), @@ -5994,8 +5948,11 @@ mod upstream_tracking_tests { let origin = bare_origin(); let clone = clone_of(&origin, &[]); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-x") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-x"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), @@ -6018,8 +5975,11 @@ mod upstream_tracking_tests { let origin = bare_origin(); let clone = clone_of(&origin, &[]); - check_out_workspace_branch(&git_runner(clone.path(), None, None), "main", "feature-new") - .unwrap(); + check_out_workspace_branch( + &git_runner(clone.path(), None, None), + &BranchCheckout::new("main", "feature-new"), + ) + .unwrap(); assert_eq!( git(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), diff --git a/apps/staged/src-tauri/src/lib.rs b/apps/staged/src-tauri/src/lib.rs index 9692936f5..4f9fd45fa 100644 --- a/apps/staged/src-tauri/src/lib.rs +++ b/apps/staged/src-tauri/src/lib.rs @@ -2599,6 +2599,7 @@ pub fn run() { session_commands::get_session_background_hold, session_runner::reconcile::reconcile_session, session_runner::reconcile::get_session_reconcile_states, + workspace_setup::failures::get_workspace_setup_errors, session_commands::delete_session, session_commands::start_branch_session, session_commands::start_or_queue_branch_session, diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs index a1d156b05..8d0a47f10 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs @@ -519,6 +519,35 @@ impl Run<'_> { Ok(self.result(&inspection, legs, warnings)) } + /// The rows of a workspace this transfer resumed say `Running` from here + /// on, whatever the transfer comes to, rather than the state it was + /// resumed from — each one once the inspection has read its clone as on + /// its branch ([`preflight::RepoPreflight::is_on_its_branch`]). + /// + /// `Running` means the clone is on the branch as well as the workstation + /// being up (`workspace_setup`), and the resume alone vouches only for + /// the second. A row whose clone was not read, or read as somewhere + /// else, keeps its status: the status pollers find its workstation + /// running and put it through branch setup, which is what checks the + /// branch out. + fn mark_resumed_rows_running( + &self, + inspection: &Inspection, + resumed: &BTreeMap>, + ) -> Result<(), Failure> { + for (source, row) in inspection.branches.iter().zip(&inspection.preflight.repos) { + let came_back = source + .workspace + .as_deref() + .is_some_and(|workspace| matches!(resumed.get(workspace), Some(None))); + if came_back && row.is_on_its_branch() { + self.store + .update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Running)?; + } + } + Ok(()) + } + /// The preflight, again, now that the transfer is registered — with any /// suspended, stopped or failed source workspace resumed first, so its /// checkouts can be read. One that won't come back blocks the transfer. @@ -557,16 +586,8 @@ impl Run<'_> { .err(); self.guard_not_cancelled()?; let failed = error.is_some(); - match &error { - Some(e) => log::warn!("[project_transfer] cannot resume {workspace}: {e}"), - // Running now, whatever the transfer comes to: the rows - // say so, rather than the state it was resumed from. - None => { - self.store.update_workspace_status_by_workspace_name( - &workspace, - &WorkspaceStatus::Running, - )?; - } + if let Some(e) = &error { + log::warn!("[project_transfer] cannot resume {workspace}: {e}"); } resumed.insert(workspace, error); // One is enough to block the transfer: spare the others. @@ -577,6 +598,7 @@ impl Run<'_> { inspection = preflight::inspect_after_resuming( self.store, self.host, project_id, &resumed, &observer, )?; + self.mark_resumed_rows_running(&inspection, &resumed)?; } let preflight = &inspection.preflight; if !preflight.can_transfer { diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs index b1b5dd841..76340af70 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs @@ -2646,6 +2646,37 @@ fn a_resume_honours_cancel_and_leaves_the_rows_running() { assert_eq!(status(), Some(WorkspaceStatus::Running), "resumed"); } +/// A resume vouches for the workstation being up, not for what its clone +/// has checked out, and `Running` means both. A row whose clone the +/// transfer's own inspection read as on another branch keeps its status: +/// the status pollers find the workstation running and put the branch +/// through setup, which is what checks it out. +#[test] +fn a_resumed_row_whose_clone_is_on_another_branch_is_not_marked_running() { + let f = Fixture::new(&[(WIDGETS, "feature")]); + f.transfer(true).result.expect("to remote"); + git(&f.checkout(0), &["checkout", "-b", "elsewhere"]); + f.host.set_state(&f.workspace(), WorkspaceState::Suspended); + f.store + .update_workspace_status_by_workspace_name(&f.workspace(), &WorkspaceStatus::Suspended) + .unwrap(); + + let err = f.transfer(true).result.unwrap_err(); + + assert!(f + .host + .calls() + .contains(&format!("resume {}", f.workspace()))); + assert!(err.contains("has elsewhere checked out"), "{err}"); + let status = f + .store + .get_branch(&f.branches[0].id) + .unwrap() + .unwrap() + .workspace_status; + assert_eq!(status, Some(WorkspaceStatus::Suspended)); +} + // --------------------------------------------------------------------------- // Panics // --------------------------------------------------------------------------- diff --git a/apps/staged/src-tauri/src/project_transfer/preflight.rs b/apps/staged/src-tauri/src/project_transfer/preflight.rs index 8c2c5cbdb..f467d1df3 100644 --- a/apps/staged/src-tauri/src/project_transfer/preflight.rs +++ b/apps/staged/src-tauri/src/project_transfer/preflight.rs @@ -169,6 +169,22 @@ pub struct RepoPreflight { pub scanning: bool, } +impl RepoPreflight { + /// Whether the inspection read this row's checkout and found it on the + /// row's branch: there was a checkout to read, it has a commit, and HEAD + /// was neither detached nor on another branch. + pub(super) fn is_on_its_branch(&self) -> bool { + self.state == RepoTransferState::Carry + && self.head.is_some() + && !self.blockers.iter().any(|issue| { + matches!( + issue.code.as_str(), + "not_a_checkout" | "detached_head" | "wrong_branch" | "no_commits" + ) + }) + } +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct ProjectTransferPreflight { diff --git a/apps/staged/src-tauri/src/prs/commit.rs b/apps/staged/src-tauri/src/prs/commit.rs index b566019bc..73fd03a13 100644 --- a/apps/staged/src-tauri/src/prs/commit.rs +++ b/apps/staged/src-tauri/src/prs/commit.rs @@ -13,7 +13,10 @@ use crate::session_commands::{fail_claimed_queued_start, QueuedSessionClaim, Que use crate::session_runner; use crate::store::{self, FailureStrategy, PipelineExecution, PipelineKind, PipelineStep, Store}; -use super::context::{base_branch_name, resolve_branch_pipeline_context, BranchPipelineContext}; +use super::context::{ + base_branch_name, ensure_pipeline_checkout, resolve_branch_pipeline_context, + BranchPipelineContext, +}; use super::git_commands::git_fetch_with_fallback; use super::pipeline::{ branch_has_work_in_flight, find_queued_pipeline, launch_running_pipeline_session, @@ -410,6 +413,9 @@ pub(crate) async fn start_or_queue_commit_pipeline_for_branch( } let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // The branch is idle, so the pipeline runs now. Held until this returns, + // by when its row is `running`. + let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None).await?; let base_branch = base_branch_name(&ctx.branch).to_string(); let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let rebase_ref = rebase_ref_for_target(&ctx.branch, &remote_branch, target.as_deref()); @@ -510,6 +516,10 @@ pub(crate) async fn start_queued_commit_pipeline_for_branch( .map(|_| "origin"); let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // Before the claim, like everything else here that awaits. Held until + // this returns, by when the row is claimed `running`. + let _checkout = + ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, Some(&session.id)).await?; let base_branch = base_branch_name(&ctx.branch).to_string(); let remote_branch = if target.is_some() { crate::branches::resolve_remote_branch(&store, &ctx.branch).await diff --git a/apps/staged/src-tauri/src/prs/context.rs b/apps/staged/src-tauri/src/prs/context.rs index 2cbd9f096..555ad4e6f 100644 --- a/apps/staged/src-tauri/src/prs/context.rs +++ b/apps/staged/src-tauri/src/prs/context.rs @@ -101,6 +101,32 @@ pub(crate) fn resolve_branch_pipeline_context( }) } +/// A pipeline's share of the checkout check every start on a remote branch +/// makes (`workspace_setup::verify`): the clone its git steps run in is read +/// to be on the branch, and put there when it is not. A rebase, squash or +/// commit in a clone that is on the base branch rewrites the base; a push +/// from one sends whatever the local branch last held. A no-op for a local +/// branch. +/// +/// Call it once the pipeline is known to run now, before its row goes +/// `running`, and keep what it returns until it has +/// (`workspace_setup::verify::CheckoutHold`). `session_id` is the pipeline's +/// row when it already exists (a queued one, a pull's marker). +pub(crate) async fn ensure_pipeline_checkout( + app_handle: &tauri::AppHandle, + store: &Arc, + branch: &store::Branch, + session_id: Option<&str>, +) -> Result { + crate::workspace_setup::verify::ensure_session_checkout( + app_handle, + store, + branch, + session_id.unwrap_or_default(), + ) + .await +} + /// Derive the base branch name, stripping the `origin/` prefix if present. /// /// GitHub's PR API expects the bare branch name, while local comparison diff --git a/apps/staged/src-tauri/src/prs/create_pr.rs b/apps/staged/src-tauri/src/prs/create_pr.rs index eb93b076a..a48132496 100644 --- a/apps/staged/src-tauri/src/prs/create_pr.rs +++ b/apps/staged/src-tauri/src/prs/create_pr.rs @@ -7,7 +7,7 @@ use crate::git; use crate::session_runner; use crate::store::{FailureStrategy, PipelineStep, Store}; -use super::context::{base_branch_name, resolve_branch_pipeline_context}; +use super::context::{base_branch_name, ensure_pipeline_checkout, resolve_branch_pipeline_context}; use super::git_commands::{git_fetch_with_fallback, git_push_with_fallback, push_refspec}; use super::pipeline::start_pipeline_for_branch; @@ -126,6 +126,8 @@ pub(crate) async fn start_create_pr_pipeline_for_branch( ) -> Result { crate::project_transfer::ensure_branch_not_transferring(&branch_id)?; let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // Held until this returns, by when the pipeline's row is `running`. + let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None).await?; let base_branch = base_branch_name(&ctx.branch); let is_draft = draft.unwrap_or(false); diff --git a/apps/staged/src-tauri/src/prs/git_pipeline.rs b/apps/staged/src-tauri/src/prs/git_pipeline.rs index d0ed842d6..ceb10afe4 100644 --- a/apps/staged/src-tauri/src/prs/git_pipeline.rs +++ b/apps/staged/src-tauri/src/prs/git_pipeline.rs @@ -11,7 +11,7 @@ use crate::session_commands::{fail_claimed_queued_start, QueuedSessionClaim, Que use crate::session_runner; use crate::store::{self, PipelineExecution, PipelineKind, PipelineStep, Store}; -use super::context::resolve_branch_pipeline_context; +use super::context::{ensure_pipeline_checkout, resolve_branch_pipeline_context}; use super::pipeline::pipeline_prompt; use super::pull::build_pull_pipeline_steps; use super::push::{build_push_pipeline_steps, PushVariant}; @@ -91,6 +91,10 @@ pub(crate) async fn start_queued_git_pipeline_for_branch( .ok_or_else(|| format!("Queued session {} has no pipeline", session.id))?; let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // Before the claim, like everything else here that awaits. Held until + // this returns, by when the row is claimed `running`. + let _checkout = + ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, Some(&session.id)).await?; let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let RebuiltGitPipeline { steps, diff --git a/apps/staged/src-tauri/src/prs/pull.rs b/apps/staged/src-tauri/src/prs/pull.rs index 24a683866..7f7671d96 100644 --- a/apps/staged/src-tauri/src/prs/pull.rs +++ b/apps/staged/src-tauri/src/prs/pull.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use crate::session_runner; use crate::store::{self, FailureStrategy, PipelineExecution, PipelineKind, PipelineStep, Store}; +use super::context::ensure_pipeline_checkout; use super::git_commands::git_fetch_with_fallback; use super::pipeline::{branch_has_work_in_flight, find_queued_pipeline, PULL_PROMPT}; @@ -183,10 +184,33 @@ pub(crate) async fn pull_or_queue_branch_for_branch( let pull_store = Arc::clone(&store); let pull_branch_id = branch_id.clone(); let kick_branch_id = branch_id.clone(); + let verify_store = Arc::clone(&store); + let verify_app_handle = app_handle.clone(); + let verify_branch_id = branch_id.clone(); pull_or_queue_branch_with( store, registry, branch_id, + // The fast-forward merges into whatever the clone has checked out, + // so a remote clone is read to be on the branch first, like a + // pipeline's. The marker row is `running` by now and is what keeps + // other checkouts out of the clone, so the hold ends with the read. + move |marker_session_id| async move { + let Some(branch) = verify_store + .get_branch(&verify_branch_id) + .map_err(|e| e.to_string())? + else { + return Ok(()); + }; + ensure_pipeline_checkout( + &verify_app_handle, + &verify_store, + &branch, + Some(&marker_session_id), + ) + .await + .map(drop) + }, move || crate::timeline::pull_branch_ff_only_impl(&pull_store, &pull_branch_id), move || { // Anything the user requested while the pull held the branch is @@ -202,15 +226,21 @@ pub(crate) async fn pull_or_queue_branch_for_branch( .await } -/// [`pull_or_queue_branch_for_branch`] with the pull itself and the -/// after-release hook injected, so a test can stand in a pull that blocks. -pub(super) async fn pull_or_queue_branch_with( +/// [`pull_or_queue_branch_for_branch`] with the checkout check, the pull +/// itself and the after-release hook injected, so a test can stand in a pull +/// that blocks. `verify` is given the marker session's id and runs before +/// the pull; its error is the pull's. +pub(super) async fn pull_or_queue_branch_with( store: Arc, registry: Arc, branch_id: String, + verify: impl FnOnce(String) -> V + Send + 'static, pull: impl FnOnce() -> Result<(), String> + Send + 'static, on_released: impl FnOnce() + Send + 'static, -) -> Result, String> { +) -> Result, String> +where + V: std::future::Future> + Send + 'static, +{ let (session_id, liveness) = match claim_or_queue_pull_for_branch(&store, ®istry, &branch_id)? { PullDisposition::Queued(session_id) => return Ok(Some(session_id)), @@ -231,9 +261,12 @@ pub(super) async fn pull_or_queue_branch_with( // the pull's own error path rather than returning early: the marker // session is what marks the branch busy, and nothing else ever finishes it. let released = tauri::async_runtime::spawn(async move { - let pulled = tauri::async_runtime::spawn_blocking(pull) - .await - .unwrap_or_else(|e| Err(format!("Pull task failed: {e}"))); + let pulled = match verify(session_id.clone()).await { + Ok(()) => tauri::async_runtime::spawn_blocking(pull) + .await + .unwrap_or_else(|e| Err(format!("Pull task failed: {e}"))), + Err(e) => Err(e), + }; finish_immediate_pull_session( &store, diff --git a/apps/staged/src-tauri/src/prs/push.rs b/apps/staged/src-tauri/src/prs/push.rs index e71cb9b66..cd262f47a 100644 --- a/apps/staged/src-tauri/src/prs/push.rs +++ b/apps/staged/src-tauri/src/prs/push.rs @@ -13,7 +13,9 @@ use crate::git; use crate::session_runner; use crate::store::{self, FailureStrategy, PipelineExecution, PipelineKind, PipelineStep, Store}; -use super::context::{resolve_branch_pipeline_context, BranchPipelineContext}; +use super::context::{ + ensure_pipeline_checkout, resolve_branch_pipeline_context, BranchPipelineContext, +}; use super::git_commands::{git_push_with_fallback, push_refspec}; use super::pipeline::{ branch_has_work_in_flight, find_queued_pipeline, launch_running_pipeline_session, @@ -389,6 +391,9 @@ pub(crate) async fn start_or_queue_push_pipeline_for_branch( } let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // The branch is idle, so the push runs now. Held until this returns, by + // when its row is `running`. + let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None).await?; let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let steps = build_push_pipeline_steps(&ctx.branch.branch_name, &remote_branch, variant); diff --git a/apps/staged/src-tauri/src/prs/tests/pull.rs b/apps/staged/src-tauri/src/prs/tests/pull.rs index 0f17e99a4..376dcde75 100644 --- a/apps/staged/src-tauri/src/prs/tests/pull.rs +++ b/apps/staged/src-tauri/src/prs/tests/pull.rs @@ -25,6 +25,7 @@ async fn dropping_the_request_future_mid_pull_no_longer_strands_the_marker() { Arc::clone(&store), Arc::clone(®istry), branch.id.clone(), + |_marker_session_id| async { Ok(()) }, move || { released.recv().map_err(|e| e.to_string())?; Ok(()) diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index f5b38a85c..6582d2329 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -1892,7 +1892,9 @@ pub(crate) async fn resume_session_for_store( } let plan = prepare_resume(&store, &session, branch_id, acp_config_selection).await?; - ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; + // Held until this returns, by when the row is `running` and visible to + // any checkout that would otherwise run in the clone. + let _checkout = ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; // Live from before the `running` write until the runner has registered // (see `SessionRegistry::hold`); released when this function returns. @@ -1970,20 +1972,24 @@ fn ensure_claimed_resume_still_running(store: &Store, session_id: &str) -> Resul /// clone can't be read fails, since the turn would otherwise run unchecked. /// Its waits are bounded (two capped reads; a repair's execs each carry /// their own limit), so a claimed row is not held `running` without end. +/// +/// The caller keeps what this returns until the row is `running` (see +/// `workspace_setup::verify::CheckoutHold`). async fn ensure_resume_checkout( app_handle: &tauri::AppHandle, store: &Arc, plan: &ResumePlan, session_id: &str, -) -> Result<(), String> { +) -> Result { + use crate::workspace_setup::verify::CheckoutHold; if plan.workspace_name.is_none() { - return Ok(()); + return Ok(CheckoutHold::none()); } let Some(branch_id) = plan.event_branch_id.as_deref() else { - return Ok(()); + return Ok(CheckoutHold::none()); }; let Some(branch) = store.get_branch(branch_id).map_err(|e| e.to_string())? else { - return Ok(()); + return Ok(CheckoutHold::none()); }; crate::workspace_setup::verify::ensure_session_checkout(app_handle, store, &branch, session_id) .await @@ -2700,11 +2706,14 @@ pub(crate) async fn resume_claimed_session_for_store( let prepared = match prepare_resume(&store, &session, None, None).await { Ok(plan) => ensure_resume_checkout(&app_handle, &store, &plan, &session.id) .await - .map(|()| plan), + .map(|checkout| (plan, checkout)), Err(e) => Err(e), }; + // The row has been `running` since the claim, so a checkout elsewhere in + // the clone already sees this session; the hold only has to outlast the + // read. let plan = match prepared { - Ok(plan) => plan, + Ok((plan, _checkout)) => plan, Err(e) => { let branch_id = store.get_branch_id_for_session(&session.id).ok().flatten(); let project_id = store.get_project_id_for_session(&session.id).ok().flatten(); @@ -5343,9 +5352,14 @@ async fn prepare_branch_session_start( // A remote branch's clone is read to be on the branch before the context // below is built from it and an agent is started in it (a no-op for a - // local branch). See `workspace_setup::verify`. - crate::workspace_setup::verify::ensure_session_checkout(app_handle, store, branch, session_id) - .await?; + // local branch). See `workspace_setup::verify`. The immediate start's + // row is `running` already, so a checkout elsewhere in the clone sees + // this session without the hold outliving the read. + let _checkout = crate::workspace_setup::verify::ensure_session_checkout( + app_handle, store, branch, session_id, + ) + .await?; + drop(_checkout); // Remote branches use ws_exec for git operations; local branches use the // worktree directly. @@ -6900,8 +6914,9 @@ async fn start_queued_session_for_branch( // local branch; see `workspace_setup::verify`). Before the claim, like // everything else here that awaits: a read that fails leaves the row // queued for the drain's bounded retries, and a repair that fails leaves - // the branch in `Error`, where the queue holds its rows. - crate::workspace_setup::verify::ensure_session_checkout( + // the branch in `Error`, where the queue holds its rows. The clone stays + // held until this returns, by when the row is claimed `running`. + let _checkout = crate::workspace_setup::verify::ensure_session_checkout( &app_handle, &store, &branch, @@ -7204,8 +7219,9 @@ async fn start_requeued_session_for_branch( // neither holds this branch's drain open nor spends a retry. let plan = prepare_resume(&store, &session, Some(branch_id.clone()), None).await?; // Before the claim too: a failed read spends one of the drain's retries - // and leaves the row queued. - ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; + // and leaves the row queued. Held until this returns, by when the row is + // claimed `running`. + let _checkout = ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?; // Live from before the claim flips the row to `running` until // `start_session` has registered it (see `SessionRegistry::hold`). diff --git a/apps/staged/src-tauri/src/web_server.rs b/apps/staged/src-tauri/src/web_server.rs index a62c90d6a..d3767ae74 100644 --- a/apps/staged/src-tauri/src/web_server.rs +++ b/apps/staged/src-tauri/src/web_server.rs @@ -2246,17 +2246,17 @@ pub(crate) async fn dispatch( let branch_id: String = arg(&args, "branchId")?; let app_handle = app_handle.clone(); detached(async move { - crate::workspace_setup::ensure_branch_set_up( - &app_handle, - &store, - &branch_id, - crate::workspace_setup::SetupTrigger::Requested, - ) - .await?; - Ok(Value::Null) + let result = + crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id) + .await?; + Ok(serde_json::to_value(result).unwrap()) }) .await } + "get_workspace_setup_errors" => { + let errors = crate::workspace_setup::failures::get_workspace_setup_errors(); + Ok(serde_json::to_value(errors).unwrap()) + } "resume_workspace" => { let store = get_store(store_mutex)?; let workspace_name: String = arg(&args, "workspaceName")?; @@ -5086,7 +5086,7 @@ mod tests { "resume_or_queue_session_for_store(", "start_project_session_for_store(", // The shared workspace bodies: branch setup, resume, the pollers. - "workspace_setup::ensure_branch_set_up(", + "workspace_setup::request_branch_setup(", "resume_workspace_impl(", "workspace_setup::poll::", ]; diff --git a/apps/staged/src-tauri/src/web_server/blocking_guard_tests.rs b/apps/staged/src-tauri/src/web_server/blocking_guard_tests.rs index 6c6934a27..fcad3c6f5 100644 --- a/apps/staged/src-tauri/src/web_server/blocking_guard_tests.rs +++ b/apps/staged/src-tauri/src/web_server/blocking_guard_tests.rs @@ -513,6 +513,8 @@ const SYNCHRONOUS_COMMANDS: &[(&str, &str)] = &[ "session_runner/reconcile.rs", "get_session_reconcile_states", ), + // The reasons failed branch setups left behind, held in memory. + ("workspace_setup/failures.rs", "get_workspace_setup_errors"), // Per-window seeds and the updater claim. ("window_commands.rs", "take_window_seed"), ("window_commands.rs", "claim_updater_ownership"), diff --git a/apps/staged/src-tauri/src/workspace_setup.rs b/apps/staged/src-tauri/src/workspace_setup.rs index 7be516531..97d2bc29e 100644 --- a/apps/staged/src-tauri/src/workspace_setup.rs +++ b/apps/staged/src-tauri/src/workspace_setup.rs @@ -3,245 +3,201 @@ //! A remote branch is usable once two things are true: its workstation is //! running, and the workstation's clone is on the branch. Blox only knows //! about the first. This module owns the second, and with it the one write -//! that says both hold: it is the only code that moves a branch from -//! `Starting` to `Running`, and it does so only after the checkout succeeded -//! and the clone reads as on the branch ([`verify`]). A failure leaves the -//! branch in `Error` and is returned, never logged and dropped — a branch -//! marked `Running` on a workstation still on its base branch has its -//! sessions commit onto the base. +//! that says both hold: it is the only code that moves a branch that is not +//! `Running` to `Running`, and it does so only after the checkout succeeded +//! and the clone reads as on the branch ([`verify`]). A branch marked +//! `Running` on a workstation still on its base branch has its sessions +//! commit onto the base. //! -//! Setup is level-triggered. Whoever sees a `Starting` branch asks for it: +//! A failure leaves the branch in `Error`, and `Error` always has a way out: +//! the reason is recorded and announced ([`failures`]), a retry timer is set +//! ([`retry`]), and the error view's Retry asks for setup again. +//! +//! Setup is level-triggered. Whoever sees a branch that needs it asks: //! //! - the frontend, through the `start_workspace` command, for every -//! `Starting` branch it holds; -//! - the status pollers ([`poll`]), which see a running workstation whose -//! row is still `Starting` and trigger setup instead of promoting it; +//! `Starting` branch it holds and for an `Error` one the user retries; +//! - the status pollers ([`poll`]), which see a running workstation behind a +//! row that doesn't say `Running`. They never promote it: a `Starting` row +//! gets setup triggered, a `Stopped`, `Suspended` or statusless one is +//! moved to `Starting` first, and an `Error` one gets its retry timer; //! - `resume_workspace`, once the workstation is back; //! - the startup pass ([`resume_pending_setups`]), for branches a previous -//! run left in `Starting`. +//! run left in `Starting` or `Error`; +//! - a retry timer ([`SetupTrigger::Retry`]). //! -//! One more caller asks for a branch that is already `Running`: an agent -//! start that read the clone and found it off the branch +//! One more caller asks for a branch that is already `Running`: an agent or +//! pipeline start that read the clone and found it off the branch //! ([`verify::ensure_session_checkout`], [`SetupTrigger::Repair`]). //! //! All of them go through one [`guard::SetupGuard`], so a second request for //! a branch joins the run in flight, and two branches never run setup in one -//! workstation at once. The guard is per process: a second Staged instance -//! on the same database runs its own. +//! workstation at once. A caller that joined a run started for another +//! reason does not take that run's "nothing to do" for its own answer +//! ([`run_guarded`]). The guard is per process: a second Staged instance on +//! the same database runs its own. +//! +//! The checkout itself ([`checkout`]) never loses work or moves it between +//! branches, and never runs in a clone a live session is working in +//! ([`clone`]). +pub(crate) mod checkout; +pub(crate) mod clone; +pub(crate) mod failures; pub(crate) mod guard; +pub(crate) mod host; pub(crate) mod poll; +pub(crate) mod retry; #[cfg(test)] mod tests; pub(crate) mod verify; +mod workstation; use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; -use crate::blox::{self, BloxError}; use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; +use crate::PollWorkspaceResult; -use guard::SetupGuard; -use verify::{CloneHead, HeadRead}; +use checkout::CheckoutError; +use clone::{check_out_in_clone, checkout_for}; +use guard::{Patience, Progress, SetupGuard}; +use host::{blox_host, reports_booting, reports_running, SetupHost}; +use verify::CloneHead; +use workstation::{start_workstation, wait_until_running}; /// Who asked for a branch to be set up. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum SetupTrigger { /// A caller that wants the branch set up whatever its row says, short of - /// `Running`: the `start_workspace` command, or a repo joining a running - /// workspace. + /// `Running`: the `start_workspace` command (a new branch, or the user's + /// Retry on a failed one), or a repo joining a running workspace. Requested, /// Something that saw the row in `Starting` — a status poll, a resume, /// the startup pass. It acts only while the row still says so, so a /// trigger that arrives after setup settled the branch is a no-op. Observed, - /// An agent start that read the clone of a `Running` branch and found it - /// off the branch (see [`verify`]). The only trigger that acts on a - /// `Running` row, and only while the clone still reads wrong when its - /// turn comes. The row goes back to `Starting` for the run, so nothing - /// else starts on the branch meanwhile, and to `Error` if it fails. + /// A retry timer for a row in `Error` ([`retry`]). It acts only while + /// the row still says so and Blox reports the workstation running: it + /// never starts or resumes one. + Retry, + /// An agent or pipeline start that read the clone of a `Running` branch + /// and found it off the branch (see [`verify`]). The only trigger that + /// acts on a `Running` row, and only while the clone still reads wrong + /// when its turn comes. The row goes back to `Starting` for the run, so + /// nothing else starts on the branch meanwhile. Repair, } -/// How long setup waits for a workstation to report running after `sq blox -/// ws start` failed without saying why the workstation can't exist. -#[derive(Debug, Clone, Copy)] -pub(crate) struct ReadyWait { - pub(crate) timeout: Duration, - pub(crate) interval: Duration, -} - -impl ReadyWait { - /// `ws start` itself gives up after 120 s while the workstation keeps - /// booting. Five more minutes covers a slow bootstrap; past that the - /// branch fails visibly rather than waiting on a workstation that may - /// never come. - pub(crate) const DEFAULT: ReadyWait = ReadyWait { - timeout: Duration::from_secs(300), - interval: Duration::from_secs(5), - }; -} - -/// The clone a branch is checked out in, and what to put it on. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct Checkout { - pub(crate) workspace: String, - /// `home:` from [`branches::resolve_branch_clone_dir`]; `None` for - /// a branch with no project repo, whose git runs in the workstation's - /// default directory. - pub(crate) clone_dir: Option, - pub(crate) repo_slug: String, - pub(crate) base_ref: String, - pub(crate) branch_name: String, -} - -/// Everything setup does outside the database, so tests drive the routine -/// without `sq`. [`SetupHost::check_out`] is where a test substitutes a -/// [`branches::GitRunner`] on a local clone for the workstation's. -#[async_trait] -pub(crate) trait SetupHost: Send + Sync + 'static { - /// Whether the user is signed in to Blox. Without it `ws start` can hang. - async fn check_auth(&self) -> Result<(), BloxError>; - - /// The status Blox reports for the workstation, as `ws info` words it. - async fn workspace_status(&self, workspace: &str) -> Result, BloxError>; - - /// `sq blox ws start `; returns once bootstrap is - /// done. - async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError>; - - /// Make sure the clone exists and put it on the branch - /// ([`branches::check_out_workspace_branch`]). - async fn check_out(&self, checkout: &Checkout) -> Result<(), String>; - - /// What the clone has checked out: one cheap read, no fetch - /// ([`verify::read_clone_head`]). `Err` when the read didn't answer, - /// which says nothing about the clone. - async fn clone_head(&self, checkout: &Checkout) -> Result; - - /// Called with what [`Store::update_branch_workspace_status`] reported - /// for the `Running` write (see [`branches::workspace_status_written`]). - fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus); - - fn ready_wait(&self) -> ReadyWait { - ReadyWait::DEFAULT - } - - fn head_read(&self) -> HeadRead { - HeadRead::DEFAULT +impl SetupTrigger { + /// Whether this trigger may create or boot a workstation that isn't + /// running. Only the ones that stand for a branch being provisioned: a + /// retry waits for a running one, and a repair of a `Running` branch + /// was asked for by a session that had just read its clone. + fn may_start_workstation(self) -> bool { + matches!(self, SetupTrigger::Requested | SetupTrigger::Observed) } } -/// The real host: `sq blox` and the workstation's own git. -struct BloxHost { - app_handle: tauri::AppHandle, +/// One request for setup: why, and on whose behalf. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SetupRequest { + pub(crate) trigger: SetupTrigger, + /// The session a repair was asked for by. It may already be `running` + /// (a claimed resume), and is not a reason to leave the clone alone. + pub(crate) session: Option, } -#[async_trait] -impl SetupHost for BloxHost { - async fn check_auth(&self) -> Result<(), BloxError> { - branches::run_blox_blocking(blox::check_auth).await - } - - async fn workspace_status(&self, workspace: &str) -> Result, BloxError> { - let name = workspace.to_string(); - let info = branches::run_blox_blocking(move || blox::ws_info(&name)).await?; - if let Some(ws_id) = info.workstation_id { - if let Ok(mut cache) = branches::workstation_id_cache().lock() { - cache.insert(workspace.to_string(), ws_id); - } +impl SetupRequest { + pub(crate) fn new(trigger: SetupTrigger) -> Self { + Self { + trigger, + session: None, } - Ok(info.status) - } - - async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError> { - let name = workspace.to_string(); - let source = source.to_string(); - branches::run_blox_blocking(move || { - blox::ws_start( - &name, - Some(&source), - Some(branches::WORKSPACE_IDLE_TIMEOUT_MINUTES), - ) - }) - .await - .map(|_| ()) } - async fn check_out(&self, checkout: &Checkout) -> Result<(), String> { - match checkout.clone_dir.as_deref() { - Some(clone_dir) => { - branches::clone_repo_into_workspace( - &checkout.workspace, - clone_dir, - &checkout.repo_slug, - &checkout.base_ref, - &checkout.branch_name, - ) - .await - } - None => branches::check_out_branch_in_workspace( - &checkout.workspace, - None, - &checkout.base_ref, - &checkout.branch_name, - ) - .await - .map_err(|e| { - format!( - "Failed to set up branch '{}' in workspace '{}': {e}", - checkout.branch_name, checkout.workspace - ) - }), + pub(crate) fn repair_for(session_id: &str) -> Self { + Self { + trigger: SetupTrigger::Repair, + session: Some(session_id.to_string()).filter(|id| !id.is_empty()), } } - - async fn clone_head(&self, checkout: &Checkout) -> Result { - verify::read_clone_head(&checkout.workspace, checkout.clone_dir.as_deref()).await - } - - fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { - branches::workspace_status_written(&self.app_handle, branch_id, changed, status); - } -} - -/// Whether Blox's status string means the workstation takes commands: the -/// same reading the pollers use, so a poll that triggers setup and the setup -/// it triggers agree on whether there is anything to start. -pub(crate) fn reports_running(blox_status: Option<&str>) -> bool { - blox_status.is_some() - && branches::map_blox_status_to_workspace_status( - blox_status, - Some(&WorkspaceStatus::Starting), - ) == WorkspaceStatus::Running } -/// Whether Blox says the workstation exists and is on its way up: being -/// created and bootstrapped, or resumed. It needs waiting for, not another -/// `ws start`. -fn reports_booting(blox_status: Option<&str>) -> bool { - blox_status.is_some_and(branches::is_blox_booting_status) +/// What a setup run did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SetupOutcome { + /// The branch is checked out, read to be so, and marked `Running`. + SetUp, + /// The run's trigger had nothing to do by the time its turn came: the + /// row had moved on, a repair's clone read right, a retry's workstation + /// wasn't running. The row is as the run found it. + NothingToDo, } /// Set `branch_id` up and mark it `Running`, joining a run already in flight -/// for it. The one entry point for every trigger. +/// for it, and wait for the result. For a caller whose next step needs the +/// branch checked out: a repo joining a running workspace. pub(crate) async fn ensure_branch_set_up( app_handle: &tauri::AppHandle, store: &Arc, branch_id: &str, trigger: SetupTrigger, ) -> Result<(), String> { - let host: Arc = Arc::new(BloxHost { - app_handle: app_handle.clone(), - }); - run_guarded(SetupGuard::global(), store, &host, branch_id, trigger).await + let host = blox_host(app_handle, store); + run_guarded( + SetupGuard::global(), + store, + &host, + branch_id, + SetupRequest::new(trigger), + Patience::ToTheEnd, + ) + .await + .map(|_| ()) +} + +/// The `start_workspace` command and web arm: ask for `branch_id` to be set +/// up and return the status its row holds. +/// +/// `running` once the branch is checked out and read to be so. `starting` +/// when the run is waiting for the workstation to boot: the call returns +/// rather than hold a caller (and one of the frontend's few setup slots) for +/// the minutes that can take. The run goes on, its outcome lands on the row, +/// and a failure's reason is announced ([`failures`]). An error is the +/// failure that left the row in `Error`. +pub(crate) async fn request_branch_setup( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_id: &str, +) -> Result { + let host = blox_host(app_handle, store); + let status = run_guarded( + SetupGuard::global(), + store, + &host, + branch_id, + SetupRequest::new(SetupTrigger::Requested), + Patience::WhileWorking, + ) + .await?; + let workspace = store + .get_branch(branch_id) + .ok() + .flatten() + .and_then(|branch| branch.workspace_name); + Ok(PollWorkspaceResult { + status: status + .unwrap_or(WorkspaceStatus::Starting) + .as_str() + .to_string(), + workstation_id: workspace.and_then(|name| branches::cached_workstation_id(&name)), + }) } /// Trigger setup for a branch seen in `Starting`, without waiting for it. -/// The outcome lands on the row: `Running`, or `Error`. +/// The outcome lands on the row: `Running`, or `Error` with its reason +/// announced. pub(crate) fn trigger_branch_setup( app_handle: &tauri::AppHandle, store: &Arc, @@ -263,8 +219,36 @@ pub(crate) fn trigger_branch_setup( }); } +/// Set the retry timer for a branch seen in `Error`, unless one is out for +/// it already ([`retry::arm`]). +pub(crate) fn arm_retry( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_id: &str, +) { + retry::arm( + SetupGuard::global(), + blox_host(app_handle, store).as_ref(), + branch_id, + ); +} + +/// A branch was put in `Error` by something other than setup: a failed +/// resume, a poll that found `sq` signed out or the workstation failed. +/// Record why, so the UI can say, and set its retry timer. +pub(crate) fn branch_failed( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_id: &str, + reason: &str, +) { + failures::set(app_handle, branch_id, Some(reason)); + arm_retry(app_handle, store, branch_id); +} + /// The startup pass: set up every remote branch a previous run left in -/// `Starting`, so recovery doesn't wait for a window to load the project. +/// `Starting`, so recovery doesn't wait for a window to load the project, +/// and set the retry timer of every one it left in `Error`. /// /// Call after interrupted transfers are recovered: a transfer's branches are /// its own to provision. @@ -283,16 +267,37 @@ pub(crate) fn resume_pending_setups( return; } log::info!( - "[workspace_setup] {} branch(es) left in Starting by a previous run", + "[workspace_setup] {} branch(es) left in Starting or Error by a previous run", pending.len() ); for branch in pending { - trigger_branch_setup(app_handle, store, &branch.id); + match startup_action(&branch) { + Some(StartupAction::SetUp) => trigger_branch_setup(app_handle, store, &branch.id), + Some(StartupAction::ArmRetry) => arm_retry(app_handle, store, &branch.id), + None => {} + } } } -/// Every remote branch whose row says `Starting`, skipping projects a -/// transfer is rewriting. +/// What the startup pass does about a branch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StartupAction { + /// Left in `Starting`: finish its setup. + SetUp, + /// Left in `Error`: retry on the timer, once its workstation is running. + ArmRetry, +} + +pub(crate) fn startup_action(branch: &store::Branch) -> Option { + match branch.workspace_status { + Some(WorkspaceStatus::Starting) => Some(StartupAction::SetUp), + Some(WorkspaceStatus::Error) => Some(StartupAction::ArmRetry), + _ => None, + } +} + +/// Every remote branch the startup pass has something to do for +/// ([`startup_action`]), skipping projects a transfer is rewriting. pub(crate) fn branches_awaiting_setup(store: &Store) -> Result, String> { let mut pending = Vec::new(); for project in store.list_projects().map_err(|e| e.to_string())? { @@ -302,7 +307,7 @@ pub(crate) fn branches_awaiting_setup(store: &Store) -> Result Result, store: &Arc, host: &Arc, branch_id: &str, - trigger: SetupTrigger, -) -> Result<(), String> { - let branch = store - .get_branch(branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))?; - // Checked before taking a slot in the guard, so a trigger with nothing - // to do never becomes the run a real request joins. - if !wants_setup(&branch, trigger) { - return Ok(()); - } - let workspace = branch - .workspace_name - .clone() - .ok_or("Branch has no workspace name")?; + request: SetupRequest, + patience: Patience, +) -> Result, String> { + let status_now = || -> Result, String> { + Ok(store + .get_branch(branch_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))? + .workspace_status) + }; + for _ in 0..MAX_JOINED_ROUNDS { + let branch = store + .get_branch(branch_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + // Checked before taking a slot in the guard, so a trigger with + // nothing to do never becomes the run a real request joins. + if !wants_setup(&branch, request.trigger) { + return Ok(branch.workspace_status); + } + let workspace = branch + .workspace_name + .clone() + .ok_or("Branch has no workspace name")?; + + // A request can be answered before its run has had its turn, when + // the workstation is busy with another branch + // ([`Patience::WhileWorking`]). The row says `Starting` from now, + // not from whenever the turn comes: a window told `starting` and + // then polling a row that still read `Error` or `Stopped` would stop + // watching a branch that is being set up. Nothing awaits between + // this write and the run being in flight, and a `Starting` row is + // one every recovery path picks up. + if request.trigger == SetupTrigger::Requested + && branch.workspace_status != Some(WorkspaceStatus::Starting) + { + crate::project_transfer::ensure_branch_not_transferring(branch_id)?; + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) + .map_err(|e| e.to_string())?; + } - let store = Arc::clone(store); - let host = Arc::clone(host); - let id = branch_id.to_string(); - guard - .run(&workspace, branch_id, move || async move { - set_up_branch(&store, host.as_ref(), &id, trigger).await - }) - .await + let end = { + let guard_for_run = Arc::clone(guard); + let store = Arc::clone(store); + let host = Arc::clone(host); + let id = branch_id.to_string(); + let request = request.clone(); + guard + .run( + &workspace, + branch_id, + patience, + move |progress| async move { + set_up_branch( + &guard_for_run, + &store, + host.as_ref(), + &id, + &request, + &progress, + ) + .await + }, + ) + .await + }; + match end.result { + // Left with the run still waiting, for the workstation or for + // its turn on it. A run in flight is a branch that is starting, + // whether or not its turn has come and the row says so yet. + None => return Ok(Some(WorkspaceStatus::Starting)), + Some(Err(e)) => return Err(e), + Some(Ok(SetupOutcome::SetUp)) => return status_now(), + // This caller's own run: nothing to do is its answer. + Some(Ok(SetupOutcome::NothingToDo)) if !end.joined => return status_now(), + // Someone else's run had nothing to do. Ask again. + Some(Ok(SetupOutcome::NothingToDo)) => {} + } + } + status_now() } /// Whether `trigger` has anything to do for a branch in this state. A /// `Running` branch is set up already, and may have a session in its clone: /// nothing touches it but a repair, which an agent start asks for after /// reading the clone off the branch. An observed trigger acts only on -/// `Starting`. A repair acts on nothing else either: a branch that has -/// stopped, or failed, since the session read its clone is not one to start -/// a workstation for from a session start. +/// `Starting`, and a retry only on `Error`. A repair acts on nothing but +/// `Running` and `Starting`: a branch that has stopped, or failed, since the +/// session read its clone is not one to start a workstation for from a +/// session start. fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { + use SetupTrigger::*; match (&branch.workspace_status, trigger) { - (Some(WorkspaceStatus::Running), SetupTrigger::Repair) => true, + (Some(WorkspaceStatus::Running), Repair) => true, (Some(WorkspaceStatus::Running), _) => false, - (Some(WorkspaceStatus::Starting), SetupTrigger::Observed | SetupTrigger::Repair) => true, - (_, SetupTrigger::Observed | SetupTrigger::Repair) => false, - (_, SetupTrigger::Requested) => true, + (Some(WorkspaceStatus::Starting), Requested | Observed | Repair) => true, + (Some(WorkspaceStatus::Error), Requested | Retry) => true, + (_, Requested) => true, + (_, Observed | Retry | Repair) => false, } } @@ -366,21 +444,28 @@ fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { /// only then mark it `Running`. /// /// One routine for a workstation that has to be created and one that is -/// already running (a resumed workstation, a repo joining a shared one, a -/// retry after a restart). Both end in -/// [`branches::check_out_workspace_branch`], which is idempotent and never -/// resets a branch the clone already has, and then in -/// [`verify::confirm_checked_out`]: `Running` is written only over a clone -/// that reads as on the branch. +/// already running (a resumed one, a repo joining a shared one, a retry +/// after a restart or a failure). Both end in [`check_out_in_clone`]: +/// `Running` is written only over a clone that reads as on the branch. /// -/// Every failure past the first lookups leaves the row in `Error` and is -/// returned. Callers go through [`run_guarded`]. +/// Every failure past the first lookups is returned, and leaves the row in +/// `Error` with its reason recorded and a retry timer set. One exception: a +/// repair of a `Running` branch that was refused (a worktree with changes in +/// the way, commits that would be left behind, a live session in the clone) +/// puts the row back to `Running`. The branch was usable before the session +/// that asked came along and is exactly as usable now; only that session +/// fails, with the reason. +/// +/// Callers go through [`run_guarded`]. pub(crate) async fn set_up_branch( + guard: &Arc, store: &Arc, host: &dyn SetupHost, branch_id: &str, - trigger: SetupTrigger, -) -> Result<(), String> { + request: &SetupRequest, + progress: &Progress, +) -> Result { + let trigger = request.trigger; // A transfer provisions its own branches; refusing leaves the row alone. crate::project_transfer::ensure_branch_not_transferring(branch_id)?; @@ -391,7 +476,21 @@ pub(crate) async fn set_up_branch( // Read again now that the workstation's turn has come: the run ahead of // this one may have settled the branch. if !wants_setup(&branch, trigger) { - return Ok(()); + return Ok(SetupOutcome::NothingToDo); + } + let was_running = branch.workspace_status == Some(WorkspaceStatus::Running); + + // A retry goes ahead only on a workstation Blox reports running. One + // that is down, or can't be asked about, is looked at again later. + if trigger == SetupTrigger::Retry { + let reported = match branch.workspace_name.as_deref() { + Some(workspace) => host.workspace_status(workspace).await.ok().flatten(), + None => None, + }; + if !reports_running(reported.as_deref()) { + retry::schedule(guard, host, branch_id); + return Ok(SetupOutcome::NothingToDo); + } } // A repair was asked for by a session that read the clone off the @@ -400,14 +499,13 @@ pub(crate) async fn set_up_branch( // right now there is nothing to repair, and a `Running` branch is left // exactly as it is. A read that fails here settles nothing: the repair // goes ahead and ends in its own read. - if trigger == SetupTrigger::Repair && branch.workspace_status == Some(WorkspaceStatus::Running) - { + if trigger == SetupTrigger::Repair && was_running { if let Ok(checkout) = checkout_for(store, &branch) { if matches!( host.clone_head(&checkout).await, Ok(CloneHead::Branch(name)) if name == checkout.branch_name ) { - return Ok(()); + return Ok(SetupOutcome::NothingToDo); } } } @@ -422,69 +520,83 @@ pub(crate) async fn set_up_branch( .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) .map_err(|e| e.to_string())?; } - - let result = provision_and_check_out(store, host, &branch).await; - match &result { + host.failure_changed(branch_id, None); + + // A repair of a row that is not `Running` is the setup that row was + // waiting for, and provisions like one. + let may_start_workstation = + trigger.may_start_workstation() || (trigger == SetupTrigger::Repair && !was_running); + let provisioned = provision_and_check_out( + guard, + store, + host, + &branch, + request, + may_start_workstation, + progress, + ) + .await; + let error = match provisioned { Ok(()) => { let changed = store .update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) .map_err(|e| e.to_string())?; host.status_written(branch_id, changed, &WorkspaceStatus::Running); + guard.forget_retries(branch_id); log::info!( "[workspace_setup] branch={branch_id} checked out '{}' and running", branch.branch_name ); + return Ok(SetupOutcome::SetUp); } - Err(e) => { - log::warn!("[workspace_setup] branch={branch_id} setup failed: {e}"); - // Only over `Starting`. A poll that saw the workstation - // suspended or gone while this ran has already said something - // truer than `Error`, and with a way forward (Resume). - let still_starting = matches!( - store.get_branch(branch_id), - Ok(Some(row)) if row.workspace_status == Some(WorkspaceStatus::Starting) - ); - if still_starting { - store - .update_branch_workspace_status(branch_id, &WorkspaceStatus::Error) - .ok(); + Err(error) => error, + }; + + log::warn!("[workspace_setup] branch={branch_id} setup failed: {error}"); + // Only over `Starting`. A poll that saw the workstation suspended or + // gone while this ran has already said something truer than `Error`, + // and with a way forward (Resume). + let still_starting = matches!( + store.get_branch(branch_id), + Ok(Some(row)) if row.workspace_status == Some(WorkspaceStatus::Starting) + ); + if still_starting { + if trigger == SetupTrigger::Repair && was_running && error.is_refusal() { + // Nothing was touched, so the branch is what it was. The write + // kicks the session queue, which held while the row said + // `Starting`. + if let Ok(changed) = + store.update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) + { + host.status_written(branch_id, changed, &WorkspaceStatus::Running); } + } else { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Error) + .ok(); + host.failure_changed(branch_id, Some(error.message())); + retry::schedule(guard, host, branch_id); } } - result -} - -/// Where `branch` is checked out and what its clone should be on, from its -/// row and its project's. -pub(crate) fn checkout_for(store: &Arc, branch: &store::Branch) -> Result { - let project = store - .get_project(&branch.project_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; - let workspace = branch - .workspace_name - .as_deref() - .ok_or("Branch has no workspace name")?; - Ok(Checkout { - workspace: workspace.to_string(), - clone_dir: branches::resolve_branch_clone_dir(store, branch)?, - repo_slug: branches::resolve_branch_repo_slug(store, &project, branch)?, - base_ref: branches::normalize_branch_ref(&branch.base_branch), - branch_name: branch.branch_name.clone(), - }) + Err(error.message().to_string()) } async fn provision_and_check_out( + guard: &Arc, store: &Arc, host: &dyn SetupHost, branch: &store::Branch, -) -> Result<(), String> { - let checkout = checkout_for(store, branch)?; + request: &SetupRequest, + may_start_workstation: bool, + progress: &Progress, +) -> Result<(), CheckoutError> { + let failed = CheckoutError::Failed; + let checkout = checkout_for(store, branch).map_err(failed)?; let workspace = checkout.workspace.as_str(); // Pre-flight: without it `blox ws start` can hang or fail opaquely, // leaving the frontend spinning for minutes (issue #99). - host.check_auth().await.map_err(|e| e.to_string())?; + host.check_auth().await.map_err(|e| failed(e.to_string()))?; // A workstation that is already up — a resumed one, a shared one this // repo joins, one a previous run started — is not started again, and @@ -499,108 +611,31 @@ async fn provision_and_check_out( "[workspace_setup] branch={} workspace={workspace} is already booting, waiting for it", branch.id ); - wait_until_running(store, host, branch, workspace, "it was still booting").await?; + wait_until_running( + store, + host, + branch, + workspace, + "it was still booting", + progress, + ) + .await + .map_err(failed)?; + } else if may_start_workstation { + start_workstation(store, host, branch, &checkout, progress) + .await + .map_err(failed)?; } else { - start_workstation(store, host, branch, &checkout).await?; + // A repair or a retry found the workstation down. The clone is as + // it was, and what the row should say about a workstation that + // isn't running is the pollers' to settle. + return Err(CheckoutError::Refused(format!( + "Workspace '{workspace}' is not running (Blox reports it as {}), so branch '{}' \ + could not be checked out in it", + reported.as_deref().unwrap_or("unknown"), + checkout.branch_name + ))); } - host.check_out(&checkout).await?; - - // The checkout said it worked. `Running` rests on where the clone is, - // so that is read, not inferred. - verify::confirm_checked_out(host, &checkout).await -} - -/// `sq blox ws start`, and when that fails without ruling the workstation -/// out, a bounded wait for it to come up anyway. -async fn start_workstation( - store: &Arc, - host: &dyn SetupHost, - branch: &store::Branch, - checkout: &Checkout, -) -> Result<(), String> { - let workspace = checkout.workspace.as_str(); - let source = format!( - "https://github.com/{}.git?ref={}", - checkout.repo_slug, checkout.base_ref - ); - log::info!( - "[workspace_setup] branch={} workspace={workspace} starting from {source}", - branch.id - ); - let start_error = match host.start_workspace(workspace, &source).await { - Ok(()) => return Ok(()), - Err(BloxError::NotAuthenticated) => { - return Err("Not authenticated with Blox. Run: sq login".to_string()); - } - // Definitive: keep the CLI's text, which carries the onboarding URL - // and the action it wants. - Err(e) if branches::is_blox_onboarding_precondition_error(&e) => { - return Err(e.to_string()); - } - Err(e) => e, - }; - - // `ws start` can fail — its own 120 s limit, a dropped connection — after - // Blox created the workstation, which then keeps booting. Returning here - // would leave the branch with a running workstation and no checkout, so - // wait for it and carry on, or fail where the user can see it. - log::warn!( - "[workspace_setup] branch={} workspace={workspace} ws start failed, waiting for the workstation: {start_error}", - branch.id - ); - wait_until_running( - store, - host, - branch, - workspace, - &format!("`sq blox ws start` failed: {start_error}"), - ) - .await -} - -/// Wait, within [`SetupHost::ready_wait`], for the workstation to report -/// running. `why` says what the wait was for, for the error when it never -/// does. -async fn wait_until_running( - store: &Arc, - host: &dyn SetupHost, - branch: &store::Branch, - workspace: &str, - why: &str, -) -> Result<(), String> { - let wait = host.ready_wait(); - let started = tokio::time::Instant::now(); - loop { - match host.workspace_status(workspace).await { - Ok(status) if reports_running(status.as_deref()) => return Ok(()), - Ok(status) - if branches::map_blox_status_to_workspace_status( - status.as_deref(), - Some(&WorkspaceStatus::Starting), - ) == WorkspaceStatus::Error => - { - return Err(format!( - "Workspace '{workspace}' failed to start (Blox reports it as {}); {why}", - status.as_deref().unwrap_or("error") - )); - } - Err(BloxError::NotAuthenticated) => { - return Err("Not authenticated with Blox. Run: sq login".to_string()); - } - // Still booting, or not listed yet. - Ok(_) | Err(_) => {} - } - // Deleted while it waited: nothing left to set up. - if store.get_branch(&branch.id).ok().flatten().is_none() { - return Err(format!("Branch not found: {}", branch.id)); - } - if started.elapsed() >= wait.timeout { - return Err(format!( - "Workspace '{workspace}' did not come up within {}s; {why}", - wait.timeout.as_secs() - )); - } - tokio::time::sleep(wait.interval).await; - } + check_out_in_clone(guard, store, host, &checkout, request).await } diff --git a/apps/staged/src-tauri/src/workspace_setup/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/checkout.rs new file mode 100644 index 000000000..71a45b56e --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/checkout.rs @@ -0,0 +1,420 @@ +//! Putting a workstation's clone on a branch without losing or moving work. +//! +//! [`check_out_workspace_branch`] is the checkout every branch setup ends in. +//! It runs in a clone that may already hold work: a branch's unpushed +//! commits, another branch of the same repository (they share the clone), +//! or commits an agent made on the base branch because the branch was never +//! checked out — the state a swallowed setup failure used to leave behind. +//! +//! So it switches only when the switch keeps everything where it is, with +//! one exception that brings work along on purpose: a clone on the base +//! branch, ahead of origin's base, whose branch exists nowhere gets the +//! branch created from where it stands, so the commits made there come with +//! it. Local base is left pointing at them, never reset or moved. +//! +//! A switch that would lose work or move it between branches is +//! [`CheckoutError::Refused`], with the clone untouched and a message that +//! names the branches and the commits. An exec that failed is +//! [`CheckoutError::Failed`]: it says nothing about the clone and is worth +//! trying again. + +use crate::branches::GitRunner; + +/// Why a checkout did not happen. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CheckoutError { + /// The clone is in a state the checkout will not switch away from, + /// because doing so would lose work or move it onto another branch. The + /// clone is as it was; trying again changes nothing until someone has + /// dealt with what is in it. + Refused(String), + /// A step failed: a fetch, a probe, the checkout itself. + Failed(String), +} + +impl CheckoutError { + pub(crate) fn is_refusal(&self) -> bool { + matches!(self, CheckoutError::Refused(_)) + } + + pub(crate) fn message(&self) -> &str { + match self { + CheckoutError::Refused(message) | CheckoutError::Failed(message) => message, + } + } + + /// The same error with its message rewritten. + pub(crate) fn map_message(self, f: impl FnOnce(String) -> String) -> Self { + match self { + CheckoutError::Refused(message) => CheckoutError::Refused(f(message)), + CheckoutError::Failed(message) => CheckoutError::Failed(f(message)), + } + } +} + +impl std::fmt::Display for CheckoutError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message()) + } +} + +/// What to put the clone on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct BranchCheckout<'a> { + pub(crate) base_ref: &'a str, + pub(crate) branch_name: &'a str, + /// Whether the base branch is itself a branch Staged works on in this + /// clone. Its local commits are then that branch's own, not ones left + /// behind on the base by mistake, and they are neither brought along + /// nor a reason to refuse. + pub(crate) base_is_worked_on: bool, +} + +impl<'a> BranchCheckout<'a> { + /// A checkout in a clone whose base branch nothing works on. + #[cfg(test)] + pub(crate) fn new(base_ref: &'a str, branch_name: &'a str) -> Self { + Self { + base_ref, + branch_name, + base_is_worked_on: false, + } + } +} + +/// How many commits a message lists before it says "and N more". +const LISTED_COMMITS: usize = 5; + +/// What the clone has checked out, as far as a checkout needs to know. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Head { + Branch(String), + Detached, +} + +impl std::fmt::Display for Head { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Head::Branch(name) => write!(f, "branch '{name}'"), + Head::Detached => f.write_str("a detached HEAD"), + } + } +} + +fn failed(what: impl std::fmt::Display, error: impl std::fmt::Display) -> CheckoutError { + CheckoutError::Failed(format!("{what}: {error}")) +} + +/// Fetch `base_ref` into the clone `git` runs in, and put the clone on +/// `branch_name`. +/// +/// A branch the clone already has is checked out as it is, never recreated. +/// Every remote branch of a repo shares one clone, and this runs again +/// whenever any of them is set up again (e.g. after `resume_workspace`), so +/// `checkout -B` would reset a branch onto origin and drop its unpushed +/// commits, whichever branch HEAD happened to be on. When origin has the +/// branch too, it is then fast-forwarded if it can be: a branch that is only +/// behind picks up what was pushed, and one that has diverged keeps its local +/// commits. Only a branch the clone doesn't have yet is created: from +/// origin's copy, from where the clone stands when that is the base branch +/// with commits origin's base doesn't have (see the module docs), or from +/// origin's base. +/// +/// Before the clone leaves what it has checked out, [`guard_leaving`] makes +/// sure nothing is lost or carried across by the switch. +pub(crate) fn check_out_workspace_branch( + git: &GitRunner<'_>, + checkout: &BranchCheckout<'_>, +) -> Result<(), CheckoutError> { + let BranchCheckout { + base_ref, + branch_name, + .. + } = *checkout; + git(&["fetch", "origin", base_ref]) + .map_err(|e| failed(format!("could not fetch base branch '{base_ref}'"), e))?; + + // If the branch already exists on the remote (e.g. from an existing PR), + // fetch it so we pick up its commits: a new branch starts from there and + // an existing one fast-forwards to it. Otherwise a new branch falls back + // to starting from the base branch. + // + // Whether it exists is read from `ls-remote`'s output, not from a fetch + // failing: a fetch that never reached origin (a dropped exec, a network + // blip) would otherwise read as "not on origin" and branch from the base, + // leaving the PR's commits behind. Any failure here is a failure. + let local_ref = format!("refs/heads/{branch_name}"); + let has_remote_branch = branch_name != base_ref + && !git(&["ls-remote", "--heads", "origin", &local_ref]) + .map_err(|e| { + failed( + format!("could not check whether '{branch_name}' is on origin"), + e, + ) + })? + .trim() + .is_empty(); + if has_remote_branch { + git(&["fetch", "origin", branch_name]) + .map_err(|e| failed(format!("could not fetch branch '{branch_name}'"), e))?; + } + let remote_branch = format!("origin/{branch_name}"); + + // Whether the clone has the branch decides between checking it out as it + // is and creating it. A failed exec is neither answer: read as + // "missing", it would recreate a branch the clone does have from the + // base, moving it off its unpushed commits. + let has_local_branch = local_branch_exists(git, branch_name).map_err(CheckoutError::Failed)?; + + let head = current_head(git)?; + let on_branch = head == Head::Branch(branch_name.to_string()); + let stranded = if on_branch { + Vec::new() + } else { + guard_leaving(git, &head, checkout)? + }; + + if has_local_branch { + if !stranded.is_empty() { + return Err(refuse_stranded(&stranded, checkout, "the clone has")); + } + if !on_branch { + switch(git, &head, branch_name, &["checkout", branch_name])?; + } + if has_remote_branch { + // Fails when the branch has diverged from origin (or local + // changes are in the way). The branch then stays as it is, local + // commits and all, which is the point: never reset it. + let _ = git(&["merge", "--ff-only", &remote_branch]); + } + return Ok(()); + } + + if has_remote_branch { + if !stranded.is_empty() { + return Err(refuse_stranded(&stranded, checkout, "origin has")); + } + // `checkout -B origin/` makes the checkout track + // the branch it pushes to. + return switch( + git, + &head, + branch_name, + &["checkout", "-B", branch_name, &remote_branch], + ); + } + + if !stranded.is_empty() { + // The branch exists nowhere and the base branch here holds commits + // origin's doesn't: work made on the base because the branch was + // never checked out. The branch is created where the clone stands, + // so that work is on it. `checkout -b` keeps the worktree as it is + // (it cannot conflict) and leaves local base where it was. + log::warn!( + "[workspace_setup] creating '{branch_name}' from local '{base_ref}', which holds {} \ + commit(s) that are on no other branch and not on origin: {}", + stranded.len(), + describe_commits(&stranded) + ); + return git(&["checkout", "--no-track", "-b", branch_name]) + .map(|_| ()) + .map_err(|e| failed(format!("could not create branch '{branch_name}'"), e)); + } + + // A fresh branch must not track `origin/` — git's upstream is read + // back as the push target. + switch( + git, + &head, + branch_name, + &[ + "checkout", + "--no-track", + "-B", + branch_name, + &format!("origin/{base_ref}"), + ], + ) +} + +/// Whether the clone `git` runs in has a local branch `branch_name`. +/// +/// Answered through stdout: `for-each-ref` exits 0 whether or not the ref +/// exists and prints it only when it does, so "missing" is an empty answer +/// from a command that ran. A probe that answers through its exit status +/// (`rev-parse --verify --quiet`) can't be told apart from an exec that never +/// reached the workstation, and here that failure is an error. +pub(crate) fn local_branch_exists(git: &GitRunner<'_>, branch_name: &str) -> Result { + let local_ref = format!("refs/heads/{branch_name}"); + let refs = git(&["for-each-ref", "--format=%(refname)", &local_ref]).map_err(|e| { + format!("could not check whether the clone already has branch '{branch_name}': {e}") + })?; + // The pattern also matches refs under `/`, so only an exact line + // counts. + Ok(refs.lines().any(|line| line.trim() == local_ref)) +} + +/// What the clone has checked out. `rev-parse --abbrev-ref HEAD` exits 0 for +/// a branch and for a detached HEAD alike (it prints `HEAD` for the latter), +/// so a failed exec is not mistaken for either. +fn current_head(git: &GitRunner<'_>) -> Result { + let out = git(&["rev-parse", "--abbrev-ref", "HEAD"]) + .map_err(|e| failed("could not read what the clone has checked out", e))?; + match out.trim() { + "" => Err(CheckoutError::Failed( + "could not read what the clone has checked out: empty answer".to_string(), + )), + "HEAD" => Ok(Head::Detached), + name => Ok(Head::Branch(name.to_string())), + } +} + +/// Make sure the clone can leave `head` for the branch without losing work +/// or carrying it across, and return the commits it would leave stranded on +/// the base branch (see the module docs) for the caller to bring along or +/// refuse over. +/// +/// - A detached HEAD holding commits no ref reaches: leaving drops them. +/// - Another branch with uncommitted changes: git would carry them onto +/// this branch, or abort. They are that branch's work. +/// - The base branch holding commits nothing else does: reported, not +/// refused here. +/// Uncommitted changes on the base are left to git, which carries them +/// along (a bootstrap that touched a lockfile, an agent's work in +/// progress) or aborts untouched ([`switch`]). +fn guard_leaving( + git: &GitRunner<'_>, + head: &Head, + checkout: &BranchCheckout<'_>, +) -> Result, CheckoutError> { + let BranchCheckout { + base_ref, + branch_name, + base_is_worked_on, + } = *checkout; + match head { + Head::Detached => { + let unreachable = ["HEAD", "--not", "--branches", "--tags", "--remotes"]; + let orphaned = list_commits(git, &unreachable)?; + if orphaned.is_empty() { + return Ok(Vec::new()); + } + Err(CheckoutError::Refused(format!( + "the clone has a detached HEAD with {} commit(s) that are on no branch ({}). \ + Checking out '{branch_name}' would leave them unreachable, so the clone was \ + left as it is. Put them on a branch in the workstation, then try again.", + orphaned.len(), + describe_commits(&orphaned) + ))) + } + // Commits that only local base holds: on no remote ref and on no + // other local branch. Once a branch has been created from them they + // are that branch's, and no longer stranded here. + Head::Branch(name) if name == base_ref && !base_is_worked_on => list_commits( + git, + &[ + "HEAD", + "--not", + "--remotes", + &format!("--exclude={base_ref}"), + "--branches", + ], + ), + Head::Branch(name) => { + let changed = uncommitted_changes(git)?; + if changed == 0 { + return Ok(Vec::new()); + } + Err(CheckoutError::Refused(format!( + "the clone has branch '{name}' checked out with uncommitted changes to \ + {changed} file(s). Checking out '{branch_name}' would carry them onto it, so \ + the clone was left on '{name}'. Commit or stash them there, then try again." + ))) + } + } +} + +/// The refusal for a clone on the base branch whose local commits can't be +/// brought along by creating the branch, because the branch already exists. +fn refuse_stranded( + stranded: &[String], + checkout: &BranchCheckout<'_>, + where_it_is: &str, +) -> CheckoutError { + let BranchCheckout { + base_ref, + branch_name, + .. + } = *checkout; + CheckoutError::Refused(format!( + "the clone is on '{base_ref}' with {} commit(s) that are on no other branch and not on \ + origin ({}), and {where_it_is} a branch '{branch_name}' already, so they can't be \ + brought along by creating it. Checking out '{branch_name}' would leave them behind on '{base_ref}', so \ + the clone was left as it is. Move them onto '{branch_name}' in the workstation (or \ + reset '{base_ref}' if they are not wanted), then try again.", + stranded.len(), + describe_commits(stranded) + )) +} + +/// Run the checkout that leaves `head`. Git aborts a checkout that local +/// changes are in the way of without touching anything, so a failure over a +/// worktree that isn't clean is a refusal, not something to try again. +fn switch( + git: &GitRunner<'_>, + head: &Head, + branch_name: &str, + args: &[&str], +) -> Result<(), CheckoutError> { + let error = match git(args) { + Ok(_) => return Ok(()), + Err(e) => e, + }; + match uncommitted_changes(git) { + Ok(changed) if changed > 0 => Err(CheckoutError::Refused(format!( + "the clone has {head} checked out with uncommitted changes to {changed} file(s) \ + that are in the way of checking out '{branch_name}', so the clone was left as it \ + is ({}). Commit or stash them in the workstation, then try again.", + error.trim() + ))), + _ => Err(failed( + format!("could not check out branch '{branch_name}'"), + error, + )), + } +} + +/// How many paths `git status` reports: staged, unstaged and untracked. +/// `--no-optional-locks` keeps the read from taking `.git/index.lock`, which +/// a killed exec would leave behind. +fn uncommitted_changes(git: &GitRunner<'_>) -> Result { + let out = git(&["--no-optional-locks", "status", "--porcelain"]) + .map_err(|e| failed("could not read the clone's uncommitted changes", e))?; + Ok(out.lines().filter(|line| !line.trim().is_empty()).count()) +} + +/// The commits in `range`, newest first, as ` `. +fn list_commits(git: &GitRunner<'_>, range: &[&str]) -> Result, CheckoutError> { + let mut args = vec!["log", "--format=%h %s"]; + args.extend_from_slice(range); + let out = git(&args).map_err(|e| failed("could not list the clone's local commits", e))?; + Ok(out + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_string) + .collect()) +} + +fn describe_commits(commits: &[String]) -> String { + let listed = commits + .iter() + .take(LISTED_COMMITS) + .map(String::as_str) + .collect::>() + .join("; "); + match commits.len().saturating_sub(LISTED_COMMITS) { + 0 => listed, + more => format!("{listed}; and {more} more"), + } +} diff --git a/apps/staged/src-tauri/src/workspace_setup/clone.rs b/apps/staged/src-tauri/src/workspace_setup/clone.rs new file mode 100644 index 000000000..55a7ef4b8 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/clone.rs @@ -0,0 +1,245 @@ +//! The clone a branch is checked out in: which one it is, who else is in +//! it, and the checkout itself with the clone held. +//! +//! "The same clone" is the directory on the workstation. Every remote +//! branch whose checkout is that directory counts, whichever project it +//! belongs to: the directory comes from the repository's name alone. + +use std::sync::Arc; + +use crate::branches; +use crate::store::{self, Store}; + +use super::checkout::CheckoutError; +use super::guard::SetupGuard; +use super::host::{Checkout, SetupHost}; +use super::verify; +use super::SetupRequest; + +/// Where `branch` is checked out and what its clone should be on, from its +/// row and its project's. +pub(crate) fn checkout_for(store: &Arc, branch: &store::Branch) -> Result { + let project = store + .get_project(&branch.project_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Project not found: {}", branch.project_id))?; + let workspace = branch + .workspace_name + .as_deref() + .ok_or("Branch has no workspace name")?; + let clone_dir = branches::resolve_branch_clone_dir(store, branch)?; + let base_ref = branches::normalize_branch_ref(&branch.base_branch); + let base_is_worked_on = branches_in_clone(store, workspace, clone_dir.as_deref())? + .iter() + .any(|other| other.id != branch.id && other.branch_name == base_ref); + Ok(Checkout { + workspace: workspace.to_string(), + clone_dir, + repo_slug: branches::resolve_branch_repo_slug(store, &project, branch)?, + base_ref, + branch_name: branch.branch_name.clone(), + base_is_worked_on, + }) +} + +/// Every remote branch whose checkout is the clone at `clone_dir` on +/// `workspace`, whichever project it belongs to. The clone directory comes +/// from the repository's name alone, so the project is not what decides +/// whether two branches share one. +pub(crate) fn branches_in_clone( + store: &Store, + workspace: &str, + clone_dir: Option<&str>, +) -> Result, String> { + let mut sharing = Vec::new(); + for project in store.list_projects().map_err(|e| e.to_string())? { + for branch in store + .list_branches_for_project(&project.id) + .map_err(|e| e.to_string())? + { + if branch.branch_type == store::BranchType::Remote + && branch.workspace_name.as_deref() == Some(workspace) + && branches::resolve_branch_clone_dir(store, &branch)?.as_deref() == clone_dir + { + sharing.push(branch); + } + } + } + Ok(sharing) +} + +/// The checkout, and the read that confirms it, with the clone to itself. +/// +/// The clone's gate is held from before the look for live sessions until the +/// clone has been read back ([`SetupGuard::clone_gate`]). An agent start +/// holds the same gate from its own read of the clone until its session is +/// live, so the two can't interleave: either the start is seen here as a +/// live session, or it reads the clone after this checkout is done. +/// +/// A clone a live session is working in is never touched. If it already +/// reads as on the branch there is nothing to do, and that read is the +/// verification; if not, the checkout is refused rather than run under an +/// agent, whose commits would land on the wrong branch. +pub(super) async fn check_out_in_clone( + guard: &Arc, + store: &Arc, + host: &dyn SetupHost, + checkout: &Checkout, + request: &SetupRequest, +) -> Result<(), CheckoutError> { + let failed = CheckoutError::Failed; + // Before the gate: this can wait on the reconcile service. + settle_disconnected_sessions(store, host, checkout, request).await; + + let _gate = guard + .clone_gate(&checkout.workspace, checkout.clone_dir.as_deref()) + .await; + if let Some(live) = live_session_in_clone(store, checkout, request).map_err(failed)? { + let (workspace, expected) = (&checkout.workspace, &checkout.branch_name); + return match verify::read_head(host, checkout).await { + Ok(head) if verify::is_on_branch(&head, checkout) => Ok(()), + Ok(head) => Err(CheckoutError::Refused(format!( + "Workspace '{workspace}' has {head} checked out, where branch '{expected}' was \ + expected. It was not switched because {live}; try again once that session has \ + finished." + ))), + Err(e) => Err(failed(verify::unverified(checkout, &e))), + }; + } + + host.check_out(checkout).await?; + + // The checkout said it worked. `Running` rests on where the clone is, + // so that is read, not inferred. + verify::confirm_checked_out(host, checkout) + .await + .map_err(failed) +} + +/// A session found working in a clone. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LiveSession { + pub(crate) branch_name: String, + /// `disconnected`: its turn may still be going on the workstation with + /// no Staged process attached. + pub(crate) disconnected: bool, +} + +impl std::fmt::Display for LiveSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.disconnected { + write!( + f, + "a session of branch '{}' lost its connection and may still be working in the \ + same clone (Staged is checking on it)", + self.branch_name + ) + } else { + write!( + f, + "a session of branch '{}' is running in the same clone", + self.branch_name + ) + } + } +} + +/// The sessions occupying `checkout`'s clone, besides the one the request +/// was made for: `running`, or `disconnected` with their remote turn +/// possibly still going. Each with the name of its branch. +/// +/// Scoped to the clone, not the project: every remote branch whose checkout +/// is this directory on this workstation counts ([`branches_in_clone`]), +/// this branch included. +fn sessions_in_clone( + store: &Store, + checkout: &Checkout, + request: &SetupRequest, +) -> Result, String> { + let mut found = Vec::new(); + for branch in branches_in_clone(store, &checkout.workspace, checkout.clone_dir.as_deref())? { + let busy = store + .get_busy_sessions_for_branch(&branch.id) + .map_err(|e| e.to_string())?; + for session in busy { + if request.session.as_deref() != Some(session.id.as_str()) { + found.push((branch.branch_name.clone(), session)); + } + } + } + Ok(found) +} + +/// The first live session in `checkout`'s clone, a `running` one before a +/// `disconnected` one. A checkout under a live agent would move the commits +/// it goes on to make onto another branch. +pub(crate) fn live_session_in_clone( + store: &Store, + checkout: &Checkout, + request: &SetupRequest, +) -> Result, String> { + let sessions = sessions_in_clone(store, checkout, request)?; + let live = |disconnected: bool| { + sessions + .iter() + .find(|(_, session)| { + (session.status == store::SessionStatus::Disconnected) == disconnected + }) + .map(|(branch_name, _)| LiveSession { + branch_name: branch_name.clone(), + disconnected, + }) + }; + Ok(live(false).or_else(|| live(true))) +} + +/// Give the `disconnected` sessions in `checkout`'s clone a chance to be +/// settled before they count as live. +/// +/// A `disconnected` row holds its place until the reconcile service has +/// asked the workstation what became of its turn, which it does on a backoff +/// of up to half an hour, so it could keep a clone from being repaired for +/// as long. Here the service is asked to look now, and the rows are given a +/// bounded time to leave `disconnected`: a turn that is over is settled +/// within seconds and stops counting; one that is still going is claimed +/// `running` and counts for what it is. A row still `disconnected` after +/// the wait is treated as live, and the service goes on settling it within +/// its own bounds. +async fn settle_disconnected_sessions( + store: &Arc, + host: &dyn SetupHost, + checkout: &Checkout, + request: &SetupRequest, +) { + let disconnected = |store: &Store| -> Vec { + sessions_in_clone(store, checkout, request) + .unwrap_or_default() + .into_iter() + .filter(|(_, session)| session.status == store::SessionStatus::Disconnected) + .map(|(_, session)| session.id) + .collect() + }; + let waiting_on = disconnected(store); + if waiting_on.is_empty() { + return; + } + for session_id in &waiting_on { + host.check_disconnected_session(session_id); + } + let wait = host.disconnected_session_wait(); + let started = tokio::time::Instant::now(); + while started.elapsed() < wait.timeout { + tokio::time::sleep(wait.interval).await; + if disconnected(store).is_empty() { + return; + } + } + log::info!( + "[workspace_setup] {} disconnected session(s) in the clone of '{}' on '{}' were not \ + settled within {}s", + waiting_on.len(), + checkout.branch_name, + checkout.workspace, + wait.timeout.as_secs() + ); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/failures.rs b/apps/staged/src-tauri/src/workspace_setup/failures.rs new file mode 100644 index 000000000..7dee3d990 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/failures.rs @@ -0,0 +1,71 @@ +//! Why each failed branch is in `Error`, for the UI. +//! +//! A setup's error used to reach the frontend only as the rejection of the +//! `start_workspace` call that happened to be waiting on it. A run nobody +//! was joined to — the startup pass, a poll's trigger, a retry — failed into +//! the log, and the branch showed "Something went wrong." +//! +//! So the reason is kept here and announced: every change is emitted as +//! `workspace-setup-error` (`{ branchId, error }`, `error: null` when it is +//! cleared), and [`get_workspace_setup_errors`] hands a window that mounts +//! later what it missed. +//! +//! In memory, per process: the branch row keeps only its status. After a +//! restart the reasons are gone until the next attempt fails, which the +//! startup pass's retries ([`super::retry`]) bring about on their own. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use serde::Serialize; + +/// The event a reason's change is announced with. +pub(crate) const EVENT: &str = "workspace-setup-error"; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct SetupErrorEvent<'a> { + branch_id: &'a str, + error: Option<&'a str>, +} + +fn reasons() -> &'static Mutex> { + static REASONS: OnceLock>> = OnceLock::new(); + REASONS.get_or_init(Default::default) +} + +/// Record why `branch_id` failed (`Some`) or that it no longer has a reason +/// (`None`), and tell every window and web client. Clearing a branch that +/// has no reason announces nothing. +pub(crate) fn set( + app_handle: &tauri::AppHandle, + branch_id: &str, + error: Option<&str>, +) { + let changed = { + let mut reasons = reasons().lock().unwrap_or_else(|p| p.into_inner()); + match error { + Some(error) => { + reasons.insert(branch_id.to_string(), error.to_string()); + true + } + None => reasons.remove(branch_id).is_some(), + } + }; + if changed { + crate::web_server::emit_to_all(app_handle, EVENT, SetupErrorEvent { branch_id, error }); + } +} + +/// Every branch's recorded reason. +pub(crate) fn all() -> HashMap { + reasons().lock().unwrap_or_else(|p| p.into_inner()).clone() +} + +/// The reasons the branches in `Error` failed for, by branch id, for a +/// client that missed their `workspace-setup-error` events (one mounting, or +/// reloaded). Branches that failed before this process started are absent. +#[tauri::command] +pub fn get_workspace_setup_errors() -> HashMap { + all() +} diff --git a/apps/staged/src-tauri/src/workspace_setup/guard.rs b/apps/staged/src-tauri/src/workspace_setup/guard.rs index b7aaefe5c..e7fdaa438 100644 --- a/apps/staged/src-tauri/src/workspace_setup/guard.rs +++ b/apps/staged/src-tauri/src/workspace_setup/guard.rs @@ -2,34 +2,116 @@ //! //! Setup is asked for from several places at once — the frontend's //! `start_workspace`, a status poll, a second window or web client, the -//! startup pass — and each of them means the same thing: "this branch should -//! be set up". Two checkouts in one clone collide on `.git/index.lock`, and -//! a caller that lost such a race used to report a failure for a branch that -//! was being set up fine. +//! startup pass, a retry timer, an agent start — and each of them means the +//! same thing: "this branch should be set up". Two checkouts in one clone +//! collide on `.git/index.lock`, and a caller that lost such a race used to +//! report a failure for a branch that was being set up fine. //! //! So a run is keyed by branch: a second caller for a branch whose setup is -//! in flight gets that run's result rather than starting its own. Runs for -//! different branches of one workstation take turns, since they may share a -//! clone. +//! in flight gets that run's result rather than starting its own, and is +//! told that it joined ([`RunEnd::joined`]), since the run it joined was +//! started for someone else's reason. Runs for different branches of one +//! workstation take turns, since they may share a clone. //! //! A run is its own task. A caller that goes away — a dropped web request, a //! closed window — only stops waiting; the setup still finishes and still -//! writes the branch's status. +//! writes the branch's status. A caller can also choose to stop waiting +//! while the run itself is only waiting, for its turn or for a workstation +//! to boot ([`Patience::WhileWorking`]). +//! +//! The guard holds two more things that are per process for the same +//! reason the runs are: +//! +//! - the gate of each clone ([`SetupGuard::clone_gate`]), which a setup +//! holds over its checkout and an agent start holds from reading the +//! clone until its session is live, so neither happens under the other; +//! - each failed branch's retry count and timer ticket ([`super::retry`]). use std::collections::HashMap; use std::future::Future; use std::sync::{Arc, Mutex, OnceLock}; -use tokio::sync::watch; +use tokio::sync::{watch, OwnedMutexGuard}; + +use super::SetupOutcome; + +/// Where a run is. +#[derive(Debug, Clone, PartialEq)] +enum RunState { + /// Behind another branch's run on the same workstation, which may itself + /// be waiting for the workstation. + Queued, + /// Doing something that ends on its own time: an exec, a checkout. + Working, + /// Waiting for a workstation to report running, for minutes at worst. + WaitingForWorkstation, + Done(Result), +} + +/// How long a caller stays with a run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Patience { + /// Until the run has a result. + ToTheEnd, + /// Until the run has a result or is only waiting — for its turn behind + /// another branch's run, or for a workstation to boot — whichever comes + /// first. The run goes on either way. + WhileWorking, +} + +/// What a caller of [`SetupGuard::run`] came back with. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct RunEnd { + /// Whether the run was already in flight, started by another caller for + /// its own reason. + pub(crate) joined: bool, + /// The run's result, or `None` when the caller left it waiting + /// ([`Patience::WhileWorking`]). + pub(crate) result: Option>, +} + +/// A run's way of saying what it is doing, for callers deciding whether to +/// stay ([`Patience`]). +#[derive(Clone)] +pub(crate) struct Progress { + state: Arc>, +} + +impl Progress { + pub(crate) fn waiting_for_workstation(&self) { + self.state.send_replace(RunState::WaitingForWorkstation); + } + + pub(crate) fn working(&self) { + self.state.send_replace(RunState::Working); + } +} -type Outcome = Option>; +/// One failed branch's place in the retry schedule. +#[derive(Debug, Clone, Copy, Default)] +struct RetryState { + /// How many retries have been scheduled since the branch last worked. + attempt: u32, + /// The timer that counts. An older timer firing finds a newer ticket + /// and does nothing. + ticket: u64, + /// Whether that timer is still to fire. + armed: bool, +} + +/// A clone: the workspace's name and the clone's directory on it. +type CloneKey = (String, Option); #[derive(Default)] pub(crate) struct SetupGuard { - /// Branch id → the result of the run in flight for it, once it has one. - in_flight: Mutex>>, + /// Branch id → the state of the run in flight for it. + in_flight: Mutex>>, /// Workspace name → the lock its runs take turns on. workspace_locks: Mutex>>>, + /// Clone → its gate. + clone_gates: Mutex>>>, + /// Branch id → its retry schedule. + retries: Mutex>, } impl SetupGuard { @@ -45,62 +127,166 @@ impl SetupGuard { } /// Run `setup` for `branch_id` on `workspace`, or wait for the run - /// already in flight for that branch, and return its result. + /// already in flight for that branch, and return how it ended. /// /// `setup` is not called when the caller joins. pub(crate) async fn run( self: &Arc, workspace: &str, branch_id: &str, + patience: Patience, setup: F, - ) -> Result<(), String> + ) -> RunEnd where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future> + Send + 'static, + F: FnOnce(Progress) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, { - let mut outcome = { + let (mut state, joined) = { let mut in_flight = self.lock_in_flight(); match in_flight.get(branch_id) { - Some(outcome) => outcome.clone(), + Some(state) => (state.clone(), true), None => { - let (tx, rx) = watch::channel(None); - in_flight.insert(branch_id.to_string(), rx.clone()); let turn = self.workspace_lock(workspace); + // Taken here when it is free, so a run that starts at + // once is never seen as queued. + let taken = Arc::clone(&turn).try_lock_owned().ok(); + let (tx, rx) = watch::channel(match taken { + Some(_) => RunState::Working, + None => RunState::Queued, + }); + let tx = Arc::new(tx); + in_flight.insert(branch_id.to_string(), rx.clone()); let entry = InFlight { guard: Arc::clone(self), branch_id: branch_id.to_string(), }; + let progress = Progress { + state: Arc::clone(&tx), + }; tauri::async_runtime::spawn(async move { let result = { - let _turn = turn.lock().await; - setup().await + let _turn = match taken { + Some(turn) => turn, + None => { + let turn = turn.lock_owned().await; + progress.working(); + turn + } + }; + setup(progress).await }; // Cleared before the result is published: a caller // that sees the result must be able to start a fresh // run, not join a finished one. drop(entry); - let _ = tx.send(Some(result)); + tx.send_replace(RunState::Done(result)); }); - rx + (rx, false) } } }; loop { - if let Some(result) = outcome.borrow_and_update().clone() { - return result; + let seen = state.borrow_and_update().clone(); + match seen { + RunState::Done(result) => { + return RunEnd { + joined, + result: Some(result), + } + } + RunState::Queued | RunState::WaitingForWorkstation + if patience == Patience::WhileWorking => + { + return RunEnd { + joined, + result: None, + } + } + RunState::Queued | RunState::Working | RunState::WaitingForWorkstation => {} } // The sender is dropped without a result only if the run // panicked; its entry is gone with it. - if outcome.changed().await.is_err() { - return Err("Workspace setup ended without a result".to_string()); + if state.changed().await.is_err() { + return RunEnd { + joined, + result: Some(Err("Workspace setup ended without a result".to_string())), + }; } } } + /// Take the gate of the clone at `clone_dir` on `workspace`. + /// + /// Held by a setup from before it looks for live sessions in the clone + /// until its checkout is done, and by an agent start from before it + /// reads the clone until its session row is live. So a checkout never + /// lands between a start's read and its launch, and a start never reads + /// a clone that is mid-checkout. + /// + /// Lock order: a setup takes its workstation's turn first and the gate + /// second. Nothing asks for a turn while it holds a gate. + pub(crate) async fn clone_gate( + &self, + workspace: &str, + clone_dir: Option<&str>, + ) -> OwnedMutexGuard<()> { + let gate = { + let mut gates = self.clone_gates.lock().unwrap_or_else(|p| p.into_inner()); + gates.retain(|_, gate| Arc::strong_count(gate) > 1); + gates + .entry((workspace.to_string(), clone_dir.map(str::to_string))) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + gate.lock_owned().await + } + + /// Count one more retry for `branch_id` and return which attempt it is + /// (0 for the first) and the ticket its timer must present. + pub(crate) fn schedule_retry(&self, branch_id: &str) -> (u32, u64) { + let mut retries = self.lock_retries(); + let state = retries.entry(branch_id.to_string()).or_default(); + let attempt = state.attempt; + state.attempt = state.attempt.saturating_add(1); + state.ticket += 1; + state.armed = true; + (attempt, state.ticket) + } + + /// Whether a retry timer is still to fire for `branch_id`. + pub(crate) fn retry_is_armed(&self, branch_id: &str) -> bool { + self.lock_retries() + .get(branch_id) + .is_some_and(|state| state.armed) + } + + /// Called by a timer when it fires: whether it is still the one that + /// counts. Disarms the schedule when it is. + pub(crate) fn retry_fired(&self, branch_id: &str, ticket: u64) -> bool { + let mut retries = self.lock_retries(); + match retries.get_mut(branch_id) { + Some(state) if state.ticket == ticket && state.armed => { + state.armed = false; + true + } + _ => false, + } + } + + /// The branch worked, or is no longer one to retry: forget its schedule. + /// A timer still out for it finds no ticket and does nothing. + pub(crate) fn forget_retries(&self, branch_id: &str) { + self.lock_retries().remove(branch_id); + } + + fn lock_retries(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.retries.lock().unwrap_or_else(|p| p.into_inner()) + } + fn lock_in_flight( &self, - ) -> std::sync::MutexGuard<'_, HashMap>> { + ) -> std::sync::MutexGuard<'_, HashMap>> { self.in_flight.lock().unwrap_or_else(|p| p.into_inner()) } diff --git a/apps/staged/src-tauri/src/workspace_setup/host.rs b/apps/staged/src-tauri/src/workspace_setup/host.rs new file mode 100644 index 000000000..fb302d50f --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/host.rs @@ -0,0 +1,254 @@ +//! What setup does outside the database, behind a trait so tests drive the +//! routine without `sq`: Blox's word on the workstation, the checkout in its +//! clone, and the app-side effects of a run (status events, failure reasons, +//! retry timers, a nudge to the reconcile service). + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; + +use crate::blox::{self, BloxError}; +use crate::branches; +use crate::store::{Store, WorkspaceStatus}; + +use super::checkout::{BranchCheckout, CheckoutError}; +use super::failures; +use super::guard::SetupGuard; +use super::retry::{self, RetryPolicy}; +use super::verify::{self, CloneHead, HeadRead}; + +/// How long setup waits for a workstation to report running after `sq blox +/// ws start` failed without saying why the workstation can't exist. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ReadyWait { + pub(crate) timeout: Duration, + pub(crate) interval: Duration, +} + +impl ReadyWait { + /// `ws start` itself gives up after 120 s while the workstation keeps + /// booting. Five more minutes covers a slow bootstrap; past that the + /// branch fails visibly rather than waiting on a workstation that may + /// never come. It is then retried on a timer ([`retry`]), so one that + /// comes up later is still set up. + pub(crate) const DEFAULT: ReadyWait = ReadyWait { + timeout: Duration::from_secs(300), + interval: Duration::from_secs(5), + }; + + /// How long a checkout waits for a `disconnected` session in its clone + /// to be settled by the reconcile service before it treats the session + /// as still working there ([`settle_disconnected_sessions`]). + pub(crate) const DISCONNECTED_SESSION: ReadyWait = ReadyWait { + timeout: Duration::from_secs(60), + interval: Duration::from_secs(2), + }; +} + +/// The clone a branch is checked out in, and what to put it on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Checkout { + pub(crate) workspace: String, + /// `home:` from [`branches::resolve_branch_clone_dir`]; `None` for + /// a branch with no project repo, whose git runs in the workstation's + /// default directory. + pub(crate) clone_dir: Option, + pub(crate) repo_slug: String, + pub(crate) base_ref: String, + pub(crate) branch_name: String, + /// Whether another branch in this clone is the base branch itself (see + /// [`BranchCheckout::base_is_worked_on`]). + pub(crate) base_is_worked_on: bool, +} + +impl Checkout { + pub(crate) fn branch_checkout(&self) -> BranchCheckout<'_> { + BranchCheckout { + base_ref: &self.base_ref, + branch_name: &self.branch_name, + base_is_worked_on: self.base_is_worked_on, + } + } +} + +/// Everything setup does outside the database, so tests drive the routine +/// without `sq`. [`SetupHost::check_out`] is where a test substitutes a +/// [`branches::GitRunner`] on a local clone for the workstation's. +#[async_trait] +pub(crate) trait SetupHost: Send + Sync + 'static { + /// Whether the user is signed in to Blox. Without it `ws start` can hang. + async fn check_auth(&self) -> Result<(), BloxError>; + + /// The status Blox reports for the workstation, as `ws info` words it. + async fn workspace_status(&self, workspace: &str) -> Result, BloxError>; + + /// `sq blox ws start `; returns once bootstrap is + /// done. + async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError>; + + /// Make sure the clone exists and put it on the branch + /// ([`checkout::check_out_workspace_branch`]). + async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError>; + + /// What the clone has checked out: one cheap read, no fetch + /// ([`verify::read_clone_head`]). `Err` when the read didn't answer, + /// which says nothing about the clone. + async fn clone_head(&self, checkout: &Checkout) -> Result; + + /// Called with what [`Store::update_branch_workspace_status`] reported + /// for a `Running` write (see [`branches::workspace_status_written`]). + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus); + + /// The reason a branch is in `Error` changed: `Some` when a run failed + /// it, `None` when a new run began ([`failures`]). + fn failure_changed(&self, branch_id: &str, error: Option<&str>); + + /// Call [`retry::fire`] for `branch_id` with `ticket` once `delay` has + /// passed. + fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64); + + /// Ask the reconcile service to settle a `disconnected` session now. + fn check_disconnected_session(&self, session_id: &str); + + fn ready_wait(&self) -> ReadyWait { + ReadyWait::DEFAULT + } + + fn disconnected_session_wait(&self) -> ReadyWait { + ReadyWait::DISCONNECTED_SESSION + } + + fn head_read(&self) -> HeadRead { + HeadRead::DEFAULT + } + + fn retry_policy(&self) -> RetryPolicy { + RetryPolicy::DEFAULT + } +} + +/// The real host: `sq blox` and the workstation's own git. +struct BloxHost { + app_handle: tauri::AppHandle, + store: Arc, +} + +/// The host the app's setups run against. +pub(crate) fn blox_host( + app_handle: &tauri::AppHandle, + store: &Arc, +) -> Arc { + Arc::new(BloxHost { + app_handle: app_handle.clone(), + store: Arc::clone(store), + }) +} + +#[async_trait] +impl SetupHost for BloxHost { + async fn check_auth(&self) -> Result<(), BloxError> { + branches::run_blox_blocking(blox::check_auth).await + } + + async fn workspace_status(&self, workspace: &str) -> Result, BloxError> { + let name = workspace.to_string(); + let info = branches::run_blox_blocking(move || blox::ws_info(&name)).await?; + if let Some(ws_id) = info.workstation_id { + if let Ok(mut cache) = branches::workstation_id_cache().lock() { + cache.insert(workspace.to_string(), ws_id); + } + } + Ok(info.status) + } + + async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError> { + let name = workspace.to_string(); + let source = source.to_string(); + branches::run_blox_blocking(move || { + blox::ws_start( + &name, + Some(&source), + Some(branches::WORKSPACE_IDLE_TIMEOUT_MINUTES), + ) + }) + .await + .map(|_| ()) + } + + async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError> { + match checkout.clone_dir.as_deref() { + Some(clone_dir) => { + branches::clone_repo_into_workspace( + &checkout.workspace, + clone_dir, + &checkout.repo_slug, + &checkout.branch_checkout(), + ) + .await + } + None => branches::check_out_branch_in_workspace( + &checkout.workspace, + None, + &checkout.branch_checkout(), + ) + .await + .map_err(|e| { + e.map_message(|m| { + format!( + "Failed to set up branch '{}' in workspace '{}': {m}", + checkout.branch_name, checkout.workspace + ) + }) + }), + } + } + + async fn clone_head(&self, checkout: &Checkout) -> Result { + verify::read_clone_head(&checkout.workspace, checkout.clone_dir.as_deref()).await + } + + fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus) { + branches::workspace_status_written(&self.app_handle, branch_id, changed, status); + } + + fn failure_changed(&self, branch_id: &str, error: Option<&str>) { + failures::set(&self.app_handle, branch_id, error); + } + + fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { + let app_handle = self.app_handle.clone(); + let store = Arc::clone(&self.store); + let branch_id = branch_id.to_string(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(delay).await; + let host = blox_host(&app_handle, &store); + retry::fire(SetupGuard::global(), &store, &host, &branch_id, ticket).await; + }); + } + + fn check_disconnected_session(&self, session_id: &str) { + if let Err(e) = crate::session_runner::reconcile::reconcile_session(session_id.to_string()) + { + log::debug!("[workspace_setup] could not ask for a check of session {session_id}: {e}"); + } + } +} + +/// Whether Blox's status string means the workstation takes commands: the +/// same reading the pollers use, so a poll that triggers setup and the setup +/// it triggers agree on whether there is anything to start. +pub(crate) fn reports_running(blox_status: Option<&str>) -> bool { + blox_status.is_some() + && branches::map_blox_status_to_workspace_status( + blox_status, + Some(&WorkspaceStatus::Starting), + ) == WorkspaceStatus::Running +} + +/// Whether Blox says the workstation exists and is on its way up: being +/// created and bootstrapped, or resumed. It needs waiting for, not another +/// `ws start`. +pub(super) fn reports_booting(blox_status: Option<&str>) -> bool { + blox_status.is_some_and(branches::is_blox_booting_status) +} diff --git a/apps/staged/src-tauri/src/workspace_setup/poll.rs b/apps/staged/src-tauri/src/workspace_setup/poll.rs index 6881bcfad..95d772d8a 100644 --- a/apps/staged/src-tauri/src/workspace_setup/poll.rs +++ b/apps/staged/src-tauri/src/workspace_setup/poll.rs @@ -3,13 +3,23 @@ //! //! A poll reconciles a branch row with what Blox reports for its //! workstation. It owns every transition but one: it never writes `Running` -//! over `Starting`. A running workstation says nothing about whether the -//! branch is checked out in it, so a poll that finds one behind a `Starting` -//! row triggers branch setup (see the parent module) and reports `Starting` -//! until setup has written `Running` itself. Nor does it write `Running` -//! over `Error`: a setup that failed left the row there, and a client still -//! polling on a stale `starting` must be told so, not handed a branch whose -//! checkout never happened. +//! over a row that doesn't already say so. A running workstation says +//! nothing about whether the branch is checked out in it, so what a poll +//! does on finding one depends on the row: +//! +//! - `Starting`: it triggers branch setup (see the parent module) and +//! reports `Starting` until setup has written `Running` itself; +//! - `Stopped`, `Suspended` or no status (a workstation resumed outside +//! Staged, or by another window): it writes `Starting` and triggers setup, +//! so the row reaches `Running` the same way a new branch does. That also +//! covers a `Starting` row a poll moved to `Suspended` while its +//! workstation was down: its way back is through setup again; +//! - `Error`: it leaves the row, which a client still polling on a stale +//! `starting` must be told about, and makes sure a retry timer is set. +//! +//! A poll that writes `Error` itself (a signed-out `sq`, a workstation Blox +//! reports failed) records why and sets the timer too, so no `Error` is +//! without a reason or a way out. use std::collections::HashMap; use std::sync::Arc; @@ -25,11 +35,21 @@ use crate::PollWorkspaceResult; /// reported for a status write (see [`branches::workspace_status_written`]). pub(crate) type StatusWritten<'a> = Box; +/// Told a branch id and why a poll moved its row to `Error`. +pub(crate) type BranchFailed<'a> = Box; + /// What a poll does besides writing the row, injected so tests can watch it. pub(crate) struct PollEffects<'a> { pub(crate) status_written: StatusWritten<'a>, - /// Called for a `Starting` branch whose workstation is running. + /// Called for a branch whose workstation is running and whose row says + /// (or has just been made to say) `Starting`. pub(crate) trigger_setup: Box, + /// Called for an `Error` branch whose workstation is running: its retry + /// timer should be set. + pub(crate) arm_retry: Box, + /// Called with a branch id and the reason when the poll itself moved the + /// row to `Error`. + pub(crate) failed: BranchFailed<'a>, } /// What Blox said about one workstation. @@ -96,15 +116,36 @@ pub(crate) fn settle_reported( let mapped = branches::map_blox_status_to_workspace_status(reported.status, row); if mapped == WorkspaceStatus::Running { match row { + // Setup said so, over a clone it read. + Some(WorkspaceStatus::Running) => {} // The workstation is up; the branch may not be checked out in // it. Setup finds out, and is the one to say `Running`. Some(WorkspaceStatus::Starting) => { (effects.trigger_setup)(&branch.id); return Ok(WorkspaceStatus::Starting); } - // Setup failed here. Only another setup can clear that. - Some(WorkspaceStatus::Error) => return Ok(WorkspaceStatus::Error), - _ => {} + // Setup failed here. Only another setup can clear that, and the + // workstation being up is what a retry waits for. + Some(WorkspaceStatus::Error) => { + (effects.arm_retry)(&branch.id); + return Ok(WorkspaceStatus::Error); + } + // The workstation came back without Staged resuming it. Where + // its clone is now is anyone's guess, so the row goes through + // setup like a new branch's rather than straight to `Running`. + Some(WorkspaceStatus::Stopped) | Some(WorkspaceStatus::Suspended) | None => { + log::debug!( + "[workspace_poll] branch={} ws={workspace} is running behind a {} row, setting it up", + branch.id, + row.map(|s| s.as_str()).unwrap_or("statusless"), + ); + let changed = store + .update_branch_workspace_status(&branch.id, &WorkspaceStatus::Starting) + .map_err(|e| e.to_string())?; + (effects.status_written)(&branch.id, changed, &WorkspaceStatus::Starting); + (effects.trigger_setup)(&branch.id); + return Ok(WorkspaceStatus::Starting); + } } } @@ -121,6 +162,15 @@ pub(crate) fn settle_reported( .update_branch_workspace_status(&branch.id, &mapped) .map_err(|e| e.to_string())?; (effects.status_written)(&branch.id, changed, &mapped); + if mapped == WorkspaceStatus::Error && row != Some(&WorkspaceStatus::Error) { + (effects.failed)( + &branch.id, + &format!( + "Blox reports workspace '{workspace}' as {}", + reported.status.unwrap_or("failed") + ), + ); + } Ok(mapped) } @@ -136,10 +186,12 @@ pub(crate) fn settle_one( Ok(reported) => reported, Err(BloxError::NotAuthenticated) => { // Auth errors are definitive — stop polling and surface the error. + let reason = "Not authenticated with Blox. Run: sq login"; store .update_branch_workspace_status(&branch.id, &WorkspaceStatus::Error) .ok(); - return Err("Not authenticated with Blox. Run: sq login".to_string()); + (effects.failed)(&branch.id, reason); + return Err(reason.to_string()); } Err(e) => { // During initial creation, `blox ws start` may still be running @@ -358,7 +410,8 @@ pub(crate) async fn poll_all_workspace_statuses( } /// The effects a real poll has: kick the session queue on a `Running` write, -/// and hand a `Starting` branch on a running workstation to setup. +/// hand a branch on a running workstation to setup, and give a failed one +/// its reason and its retry timer. fn app_effects<'a>(store: &'a Arc, app_handle: &'a AppHandle) -> PollEffects<'a> { PollEffects { status_written: Box::new(move |branch_id, changed, status| { @@ -367,5 +420,11 @@ fn app_effects<'a>(store: &'a Arc, app_handle: &'a AppHandle) -> PollEffe trigger_setup: Box::new(move |branch_id| { super::trigger_branch_setup(app_handle, store, branch_id); }), + arm_retry: Box::new(move |branch_id| { + super::arm_retry(app_handle, store, branch_id); + }), + failed: Box::new(move |branch_id, reason| { + super::branch_failed(app_handle, store, branch_id, reason); + }), } } diff --git a/apps/staged/src-tauri/src/workspace_setup/retry.rs b/apps/staged/src-tauri/src/workspace_setup/retry.rs new file mode 100644 index 000000000..0b75b2a26 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/retry.rs @@ -0,0 +1,109 @@ +//! Retrying a failed branch on its own. +//! +//! `Error` used to be a dead end: nothing polls an `Error` row, the startup +//! pass skipped it and the pollers held it. A dropped exec during a fetch +//! was enough to leave a branch there until the user deleted it. +//! +//! Every write of `Error` is now followed by a retry timer, backed off from +//! a minute to half an hour ([`RetryPolicy`]). When it fires and the row is +//! still `Error`, setup runs with [`SetupTrigger::Retry`]: it looks at the +//! workstation first and goes ahead only when Blox reports it running. A +//! retry never starts or resumes a workstation — that is the user's call, +//! through the error view's Retry or the project's Resume — so a branch +//! whose workstation is down stays `Error` and the timer is set again. +//! +//! Timers are armed by setup itself when it fails, by the pollers when they +//! see an `Error` row behind a running workstation or write `Error` +//! themselves, by a failed resume, and by the startup pass for the rows a +//! previous run left in `Error`. A branch has at most one timer that +//! counts ([`SetupGuard::retry_fired`]). + +use std::sync::Arc; +use std::time::Duration; + +use crate::store::{Store, WorkspaceStatus}; + +use super::guard::{Patience, SetupGuard}; +use super::host::SetupHost; +use super::{run_guarded, SetupRequest, SetupTrigger}; + +/// How long to wait before each retry. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RetryPolicy { + /// The wait before the first retry; it doubles with each further one. + pub(crate) base: Duration, + pub(crate) cap: Duration, +} + +impl RetryPolicy { + /// A minute covers a dropped exec or a workstation that had not quite + /// reconnected. Half an hour keeps a branch that can't be set up (a + /// signed-out `sq`, a clone that needs someone's attention) from costing + /// more than one `ws info` in that time, with no end: the branch + /// recovers whenever its cause goes away. + pub(crate) const DEFAULT: RetryPolicy = RetryPolicy { + base: Duration::from_secs(60), + cap: Duration::from_secs(30 * 60), + }; + + /// The wait before retry number `attempt` (0 for the first). + pub(crate) fn delay(&self, attempt: u32) -> Duration { + let doubled = self + .base + .checked_mul(2u32.saturating_pow(attempt.min(16))) + .unwrap_or(self.cap); + doubled.min(self.cap) + } +} + +/// Set the retry timer for a branch that has just failed, backing off from +/// its last one. +pub(crate) fn schedule(guard: &SetupGuard, host: &dyn SetupHost, branch_id: &str) { + let (attempt, ticket) = guard.schedule_retry(branch_id); + let delay = host.retry_policy().delay(attempt); + log::info!( + "[workspace_setup] branch={branch_id} retry {} in {}s", + attempt + 1, + delay.as_secs() + ); + host.retry_later(branch_id, delay, ticket); +} + +/// Set the retry timer for a branch seen in `Error`, unless one is already +/// out for it. For what finds such a row without having failed it: a poll, +/// the startup pass. +pub(crate) fn arm(guard: &SetupGuard, host: &dyn SetupHost, branch_id: &str) { + if guard.retry_is_armed(branch_id) || guard.is_in_flight(branch_id) { + return; + } + schedule(guard, host, branch_id); +} + +/// A retry timer fired. Runs setup when the timer still counts and the row +/// still says `Error`; anything else has moved the branch on. +pub(crate) async fn fire( + guard: &Arc, + store: &Arc, + host: &Arc, + branch_id: &str, + ticket: u64, +) { + if !guard.retry_fired(branch_id, ticket) { + return; + } + let row = store.get_branch(branch_id).ok().flatten(); + if !matches!(&row, Some(branch) if branch.workspace_status == Some(WorkspaceStatus::Error)) { + guard.forget_retries(branch_id); + return; + } + // A transfer provisions its own branches. Look again once it is over. + if crate::project_transfer::is_branch_transferring(branch_id) { + schedule(guard, host.as_ref(), branch_id); + return; + } + let request = SetupRequest::new(SetupTrigger::Retry); + if let Err(e) = run_guarded(guard, store, host, branch_id, request, Patience::ToTheEnd).await { + // The run recorded the reason and set the next timer. + log::warn!("[workspace_setup] retry of branch {branch_id} failed: {e}"); + } +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests.rs b/apps/staged/src-tauri/src/workspace_setup/tests.rs index 92cbf6063..3ace341e2 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests.rs @@ -2,6 +2,13 @@ //! driven without `sq`: a [`FakeHost`] answers for Blox, and a local clone //! of a bare origin stands in for the workstation's, reached through the //! same [`branches::GitRunner`] the real checkout runs on. +//! +//! The rest is in the submodules: what the checkout does to the clone +//! ([`checkout`]), the status pollers ([`poll`]), what a caller of setup +//! gets back ([`requests`]), retries +//! and failure reasons ([`retry`]), the sessions sharing a clone +//! ([`sessions`]), the reads of the clone ([`verify`]), and that every +//! trigger and check is still called from where it should be ([`wiring`]). use std::collections::VecDeque; use std::path::{Path, PathBuf}; @@ -12,18 +19,28 @@ use std::time::Duration; use async_trait::async_trait; use tokio::sync::Semaphore; -use super::guard::SetupGuard; -use super::poll::{self, PollEffects, Reported}; +use super::checkout::{BranchCheckout, CheckoutError}; +use super::guard::{Patience, SetupGuard}; +use super::host::{Checkout, ReadyWait, SetupHost}; use super::verify::{parse_clone_head, CloneHead, HeadRead, CLONE_HEAD_PROBE_SCRIPT}; -use super::{branches_awaiting_setup, run_guarded, Checkout, ReadyWait, SetupHost, SetupTrigger}; -use crate::blox::{BloxError, WorkspaceListEntry}; +use super::{ + branches_awaiting_setup, run_guarded, startup_action, SetupOutcome, SetupRequest, SetupTrigger, + StartupAction, +}; +use crate::blox::BloxError; use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; +mod checkout; +mod poll; +mod requests; +mod retry; +mod sessions; /// The checks that the clone is on the branch: at the end of setup, at /// session start, and the checkout routine's own local-branch probe. mod verify; +mod wiring; const WORKSPACE: &str = "widgets-setup-test"; @@ -149,6 +166,17 @@ struct FakeHost { meanwhile: Option>, calls: Mutex>, written: Mutex>, + /// Every change of a branch's failure reason, in order. + failures: Mutex)>>, + /// Every retry timer asked for: branch, delay, ticket. Nothing sleeps; + /// a test fires the ones it wants. + retries: Mutex>, + /// Run when the reconcile service is asked to look at a disconnected + /// session: what the service then does about it. + on_check_disconnected: Option>, + checked_sessions: Mutex>, + /// How long a workstation that isn't running yet is waited for. + ready_wait: ReadyWait, in_checkout: AtomicUsize, most_in_checkout: AtomicUsize, } @@ -165,6 +193,14 @@ impl FakeHost { meanwhile: None, calls: Mutex::new(Vec::new()), written: Mutex::new(Vec::new()), + failures: Mutex::new(Vec::new()), + retries: Mutex::new(Vec::new()), + on_check_disconnected: None, + checked_sessions: Mutex::new(Vec::new()), + ready_wait: ReadyWait { + timeout: Duration::from_millis(200), + interval: Duration::from_millis(5), + }, in_checkout: AtomicUsize::new(0), most_in_checkout: AtomicUsize::new(0), } @@ -194,18 +230,54 @@ impl FakeHost { self.written.lock().unwrap().clone() } + /// The failure reasons announced, oldest first. + fn failures(&self) -> Vec<(String, Option)> { + self.failures.lock().unwrap().clone() + } + + /// The reason the branch is left with: the last one announced for it. + fn failure_of(&self, branch: &store::Branch) -> Option { + self.failures() + .into_iter() + .rev() + .find(|(id, _)| *id == branch.id) + .and_then(|(_, reason)| reason) + } + + fn retries(&self) -> Vec<(String, Duration, u64)> { + self.retries.lock().unwrap().clone() + } + + fn checked_sessions(&self) -> Vec { + self.checked_sessions.lock().unwrap().clone() + } + async fn wait_for_checkouts(&self, count: usize) { + self.wait_for_calls("check_out", count).await; + } + + async fn wait_for_calls(&self, call: &str, count: usize) { let deadline = std::time::Instant::now() + Duration::from_secs(10); - while self.count("check_out") < count { + while self.count(call) < count { assert!( std::time::Instant::now() < deadline, - "checkout never started" + "{call} was never called {count} time(s)" ); tokio::time::sleep(Duration::from_millis(5)).await; } } } +/// Wait for `condition`, which something running beside the test brings +/// about. +async fn eventually(what: &str, condition: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !condition() { + assert!(std::time::Instant::now() < deadline, "never: {what}"); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + #[async_trait] impl SetupHost for FakeHost { async fn check_auth(&self) -> Result<(), BloxError> { @@ -241,7 +313,7 @@ impl SetupHost for FakeHost { } } - async fn check_out(&self, checkout: &Checkout) -> Result<(), String> { + async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError> { self.calls.lock().unwrap().push("check_out"); if let Some(gate) = &self.gate { gate.acquire().await.unwrap().forget(); @@ -252,11 +324,10 @@ impl SetupHost for FakeHost { // this one would be counted. tokio::time::sleep(Duration::from_millis(20)).await; let result = match self.checkout_error { - Some(error) => Err(error.to_string()), + Some(error) => Err(CheckoutError::Failed(error.to_string())), None => branches::check_out_workspace_branch( &branches::git_runner(&self.clone, None, None), - &checkout.base_ref, - &checkout.branch_name, + &checkout.branch_checkout(), ), }; self.in_checkout.fetch_sub(1, Ordering::SeqCst); @@ -280,9 +351,37 @@ impl SetupHost for FakeHost { .push((branch_id.to_string(), changed, status.clone())); } + fn failure_changed(&self, branch_id: &str, error: Option<&str>) { + self.failures + .lock() + .unwrap() + .push((branch_id.to_string(), error.map(str::to_string))); + } + + fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { + self.retries + .lock() + .unwrap() + .push((branch_id.to_string(), delay, ticket)); + } + + fn check_disconnected_session(&self, session_id: &str) { + self.checked_sessions + .lock() + .unwrap() + .push(session_id.to_string()); + if let Some(settle) = &self.on_check_disconnected { + settle(session_id); + } + } + fn ready_wait(&self) -> ReadyWait { + self.ready_wait + } + + fn disconnected_session_wait(&self) -> ReadyWait { ReadyWait { - timeout: Duration::from_millis(200), + timeout: Duration::from_millis(100), interval: Duration::from_millis(5), } } @@ -295,15 +394,35 @@ impl SetupHost for FakeHost { } } +/// One request for setup, waited for to its end. Returns the status the +/// row holds afterwards. async fn set_up( store: &Arc, host: &Arc, branch: &store::Branch, trigger: SetupTrigger, -) -> Result<(), String> { +) -> Result, String> { let guard = Arc::new(SetupGuard::default()); + set_up_under(&guard, store, host, branch, trigger).await +} + +async fn set_up_under( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, + trigger: SetupTrigger, +) -> Result, String> { let host: Arc = host.clone(); - run_guarded(&guard, store, &host, &branch.id, trigger).await + run_guarded( + guard, + store, + &host, + &branch.id, + SetupRequest::new(trigger), + Patience::ToTheEnd, + ) + .await } // ── The setup routine ─────────────────────────────────────────────────────── @@ -638,8 +757,12 @@ fn a_failed_origin_probe_is_not_read_as_a_new_branch() { real(args) }); - let error = branches::check_out_workspace_branch(&git, "main", "feature-new").unwrap_err(); + let error = + branches::check_out_workspace_branch(&git, &BranchCheckout::new("main", "feature-new")) + .unwrap_err(); + assert!(!error.is_refusal()); + let error = error.to_string(); assert!(error.contains("is on origin"), "{error}"); assert_eq!(head_branch(clone.path()), "main"); } @@ -673,7 +796,10 @@ async fn concurrent_callers_join_one_setup() { Arc::clone(&dyn_host), branch.id.clone(), ); - tokio::spawn(async move { run_guarded(&guard, &store, &host, &id, trigger).await }) + tokio::spawn(async move { + let request = SetupRequest::new(trigger); + run_guarded(&guard, &store, &host, &id, request, Patience::ToTheEnd).await + }) }) .collect(); @@ -682,7 +808,7 @@ async fn concurrent_callers_join_one_setup() { // Room for every caller to run its own checkout, were it going to. gate.add_permits(4); for caller in callers { - assert_eq!(caller.await.unwrap(), Ok(())); + assert_eq!(caller.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); } assert_eq!(host.count("check_out"), 1); @@ -715,7 +841,8 @@ async fn joined_callers_share_a_failure() { branch.id.clone(), ); tokio::spawn(async move { - run_guarded(&guard, &store, &host, &id, SetupTrigger::Requested).await + let request = SetupRequest::new(SetupTrigger::Requested); + run_guarded(&guard, &store, &host, &id, request, Patience::ToTheEnd).await }) }) .collect(); @@ -743,27 +870,15 @@ async fn setups_in_one_workstation_take_turns() { &[Info::Status("running")], Start::Succeeds, )); - let dyn_host: Arc = host.clone(); let guard = Arc::new(SetupGuard::default()); let (a, b) = tokio::join!( - run_guarded( - &guard, - &store, - &dyn_host, - &first.id, - SetupTrigger::Requested - ), - run_guarded( - &guard, - &store, - &dyn_host, - &second.id, - SetupTrigger::Requested - ), + set_up_under(&guard, &store, &host, &first, SetupTrigger::Requested), + set_up_under(&guard, &store, &host, &second, SetupTrigger::Requested), ); - assert_eq!((a, b), (Ok(()), Ok(()))); + let running = Ok(Some(WorkspaceStatus::Running)); + assert_eq!((a, b), (running.clone(), running)); assert_eq!(host.count("check_out"), 2); assert_eq!(host.most_in_checkout.load(Ordering::SeqCst), 1); assert_eq!(status_of(&store, &first), Some(WorkspaceStatus::Running)); @@ -776,250 +891,38 @@ async fn setups_in_one_workstation_take_turns() { async fn a_panicked_setup_releases_the_branch() { let guard = Arc::new(SetupGuard::default()); - let outcome = guard - .run(WORKSPACE, "branch", || async { panic!("setup blew up") }) + let ended = guard + .run(WORKSPACE, "branch", Patience::ToTheEnd, |_| async { + panic!("setup blew up") + }) .await; - assert!(outcome.is_err()); + assert!(matches!(ended.result, Some(Err(_))), "{ended:?}"); + assert!(!ended.joined); assert!(!guard.is_in_flight("branch")); - let retry = guard.run(WORKSPACE, "branch", || async { Ok(()) }).await; - assert_eq!(retry, Ok(())); -} - -// ── The pollers ───────────────────────────────────────────────────────────── - -/// What a poll did besides writing the row. -#[derive(Default)] -struct Seen { - written: Mutex>, - triggered: Mutex>, -} - -impl Seen { - fn effects(&self) -> PollEffects<'_> { - PollEffects { - status_written: Box::new(|branch_id, changed, status| { - self.written - .lock() - .unwrap() - .push((branch_id.to_string(), changed, status.clone())); - }), - trigger_setup: Box::new(|branch_id| { - self.triggered.lock().unwrap().push(branch_id.to_string()); - }), - } - } - - fn triggered(&self) -> Vec { - self.triggered.lock().unwrap().clone() - } - - fn written(&self) -> Vec<(String, bool, WorkspaceStatus)> { - self.written.lock().unwrap().clone() - } -} - -fn listed(status: &str) -> Vec { - vec![WorkspaceListEntry { - name: WORKSPACE.to_string(), - workstation_id: Some(42), - status: Some(status.to_string()), - extra: serde_json::Value::Null, - }] -} - -fn poll_batch(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { - let settled = poll::settle_all( - store, - &[branch.id.clone()], - &listed(blox_status), - &seen.effects(), - ); - settled.results[&branch.id].status.clone() -} - -fn poll_single(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { - poll::settle_one( - store, - branch, - WORKSPACE, - Ok(Reported { - status: Some(blox_status), - workstation_id: Some(42), - }), - &seen.effects(), - ) - .unwrap() - .status -} - -/// The other half of the bug: a poll saw the workstation running and marked -/// the branch `Running` with nothing checked out. Both pollers now leave the -/// row in `Starting` and ask for setup. -#[test] -fn a_poll_triggers_setup_instead_of_promoting_a_starting_branch() { - for poll in [poll_batch, poll_single] { - for blox_status in ["running", "ready", "active"] { - let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let seen = Seen::default(); - - let reported = poll(&store, &branch, blox_status, &seen); - - assert_eq!(reported, "starting"); - assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); - assert_eq!(seen.triggered(), vec![branch.id.clone()]); - // Nothing was written, so no session queue was kicked. - assert_eq!(seen.written(), vec![]); - } - } -} - -/// A batch poll also keeps fetching bootstrap progress for such a branch. -#[test] -fn a_batch_poll_keeps_a_branch_awaiting_setup_among_the_starting() { - let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let seen = Seen::default(); - - let settled = poll::settle_all( - &store, - &[branch.id.clone()], - &listed("running"), - &seen.effects(), - ); - - assert_eq!( - settled.starting_workspaces.get(WORKSPACE), - Some(&vec![branch.id.clone()]) - ); -} - -/// A setup that failed left the row in `Error`. A client still polling on a -/// stale `starting` is told so, rather than handed a `Running` branch whose -/// checkout never happened. -#[test] -fn a_poll_does_not_clear_a_failed_setup() { - for poll in [poll_batch, poll_single] { - let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let branch = set_status(&store, &branch, WorkspaceStatus::Error); - let seen = Seen::default(); - - let reported = poll(&store, &branch, "running", &seen); - - assert_eq!(reported, "error"); - assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); - assert_eq!(seen.triggered(), Vec::::new()); - assert_eq!(seen.written(), vec![]); - } -} - -/// Every other transition is still the pollers'. -#[test] -fn a_poll_still_owns_the_other_transitions() { - use WorkspaceStatus::*; - for poll in [poll_batch, poll_single] { - for (row, blox_status, expected) in [ - (Running, "suspended", Suspended), - (Running, "stopped", Stopped), - (Running, "degraded", Error), - (Running, "running", Running), - (Starting, "suspended", Suspended), - (Starting, "failed", Error), - (Starting, "provisioning", Starting), - // Blox briefly reports a new workstation as stopped. - (Starting, "stopped", Starting), - // Resumed outside Staged: it was set up before it was suspended. - (Suspended, "running", Running), - (Stopped, "running", Running), - ] { - let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let branch = set_status(&store, &branch, row.clone()); - let seen = Seen::default(); - - let reported = poll(&store, &branch, blox_status, &seen); - - assert_eq!(reported, expected.as_str(), "{row:?} + {blox_status}"); - assert_eq!(status_of(&store, &branch), Some(expected.clone())); - assert_eq!(seen.triggered(), Vec::::new()); - let changed = row != expected; - assert_eq!( - seen.written(), - vec![(branch.id.clone(), changed, expected)], - "{row:?} + {blox_status}" - ); - } - } -} - -/// A repo joining a workstation a peer already runs on is still held at -/// `Starting` by the pollers, and left to the setup that is cloning it. -#[test] -fn a_secondary_clone_is_still_held_at_starting() { - let (store, project) = store_with_project(); - let peer = starting_branch(&store, &project, "feature-one"); - set_status(&store, &peer, WorkspaceStatus::Running); - let repo = store::ProjectRepo::new(&project.id, "acme/gadgets", "feature-two", None); - store.create_project_repo(&repo).unwrap(); - let secondary = store::Branch::new_remote(&project.id, "feature-two", "main", WORKSPACE) - .with_project_repo(&repo.id); - store.create_branch(&secondary).unwrap(); - let seen = Seen::default(); - - assert_eq!(poll::is_secondary_clone_setup(&store, &secondary), Ok(true)); - assert_eq!(poll_batch(&store, &secondary, "running", &seen), "starting"); - - assert_eq!( - status_of(&store, &secondary), - Some(WorkspaceStatus::Starting) - ); - assert_eq!(seen.triggered(), Vec::::new()); - assert_eq!(seen.written(), vec![]); -} - -/// The poll's trigger, followed through: the branch a poll refused to -/// promote ends up checked out and `Running` by setup's hand. -#[tokio::test] -async fn a_polled_starting_branch_is_set_up_and_then_reported_running() { - let (_origin, clone) = origin_and_workstation_clone(); - let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let host = Arc::new(FakeHost::new( - clone.path(), - &[Info::Status("running")], - Start::Succeeds, - )); - let seen = Seen::default(); - - assert_eq!(poll_batch(&store, &branch, "running", &seen), "starting"); - for branch_id in seen.triggered() { - assert_eq!(branch_id, branch.id); - set_up(&store, &host, &branch, SetupTrigger::Observed) - .await - .unwrap(); - } - - assert_eq!(host.count("start_workspace"), 0); - assert_eq!(head_branch(clone.path()), "feature-new"); - let branch = store.get_branch(&branch.id).unwrap().unwrap(); - assert_eq!(poll_batch(&store, &branch, "running", &seen), "running"); - assert_eq!(seen.triggered().len(), 1); + let retry = guard + .run(WORKSPACE, "branch", Patience::ToTheEnd, |_| async { + Ok(SetupOutcome::SetUp) + }) + .await; + assert_eq!(retry.result, Some(Ok(SetupOutcome::SetUp))); } // ── The startup pass ──────────────────────────────────────────────────────── /// The startup pass takes the remote branches a previous run left in -/// `Starting` and nothing else. +/// `Starting`, to set up, and the ones it left in `Error`, to retry on the +/// timer. Nothing else. #[test] -fn the_startup_pass_picks_up_only_branches_left_starting() { +fn the_startup_pass_picks_up_branches_left_starting_or_failed() { let (store, project) = store_with_project(); let left_starting = starting_branch(&store, &project, "feature-starting"); + let left_failed = starting_branch(&store, &project, "feature-error"); + set_status(&store, &left_failed, WorkspaceStatus::Error); for (name, status) in [ ("feature-running", WorkspaceStatus::Running), - ("feature-error", WorkspaceStatus::Error), ("feature-suspended", WorkspaceStatus::Suspended), + ("feature-stopped", WorkspaceStatus::Stopped), ] { let branch = starting_branch(&store, &project, name); set_status(&store, &branch, status); @@ -1028,13 +931,20 @@ fn the_startup_pass_picks_up_only_branches_left_starting() { .create_branch(&store::Branch::new(&project.id, "feature-local", "main")) .unwrap(); - let pending: Vec = branches_awaiting_setup(&store) + let mut pending: Vec<(String, Option)> = branches_awaiting_setup(&store) .unwrap() .into_iter() - .map(|branch| branch.id) + .map(|branch| (branch.id.clone(), startup_action(&branch))) .collect(); + pending.sort_by_key(|(_, action)| *action != Some(StartupAction::SetUp)); - assert_eq!(pending, vec![left_starting.id]); + assert_eq!( + pending, + vec![ + (left_starting.id, Some(StartupAction::SetUp)), + (left_failed.id, Some(StartupAction::ArmRetry)), + ] + ); } /// A run that quit between a workstation coming up and its checkout left @@ -1052,6 +962,7 @@ async fn the_startup_pass_finishes_a_setup_the_last_run_dropped() { )); for pending in branches_awaiting_setup(&store).unwrap() { + assert_eq!(startup_action(&pending), Some(StartupAction::SetUp)); set_up(&store, &host, &pending, SetupTrigger::Observed) .await .unwrap(); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs new file mode 100644 index 000000000..50ba7aa2b --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs @@ -0,0 +1,320 @@ +//! What the checkout does to a clone that is not where it should be: the +//! commits and changes in it either come along or keep the clone where it +//! is. None are left behind, moved to another branch, or reset. + +use std::path::Path; +use std::sync::Arc; + +use super::super::checkout::{local_branch_exists, BranchCheckout, CheckoutError}; +use super::super::SetupTrigger; +use super::{ + head_branch, origin_and_workstation_clone, set_up, starting_branch, status_of, + store_with_project, FakeHost, Info, Start, +}; +use crate::branches; +use crate::store::WorkspaceStatus; +use crate::test_utils::run_git_in; + +fn rev(clone: &Path, name: &str) -> String { + run_git_in(clone, &["rev-parse", name]).trim().to_string() +} + +/// Commit a new file in the clone, on whatever it has checked out. +pub(super) fn commit(clone: &Path, file: &str, subject: &str) -> String { + std::fs::write(clone.join(file), format!("{subject}\n")).unwrap(); + run_git_in(clone, &["add", file]); + run_git_in(clone, &["commit", "-m", subject]); + rev(clone, "HEAD") +} + +fn uncommitted(clone: &Path) -> Vec { + run_git_in(clone, &["status", "--porcelain"]) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn has_branch(clone: &Path, name: &str) -> bool { + local_branch_exists(&branches::git_runner(clone, None, None), name).unwrap() +} + +fn check_out(clone: &Path, checkout: &BranchCheckout<'_>) -> Result<(), CheckoutError> { + branches::check_out_workspace_branch(&branches::git_runner(clone, None, None), checkout) +} + +/// A refusal, as the text the user reads. +fn refusal(result: Result<(), CheckoutError>) -> String { + let error = result.unwrap_err(); + assert!(error.is_refusal(), "not a refusal: {error}"); + error.to_string() +} + +// ── Commits made on the base branch ───────────────────────────────────────── + +/// The state the original bug left behind: the agent ran in a clone still on +/// `main` and committed there. The branch exists nowhere, so it is created +/// where the clone stands and the commits are on it. Local `main` is not +/// reset or moved, and the new branch does not track `origin/main`. +#[test] +fn commits_stranded_on_the_base_branch_come_along_to_a_new_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let origin_main = rev(clone, "origin/main"); + commit(clone, "one.txt", "agent work one"); + let stranded = commit(clone, "two.txt", "agent work two"); + + check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + + assert_eq!(head_branch(clone), "feature-new"); + assert_eq!(rev(clone, "HEAD"), stranded); + assert_eq!(rev(clone, "main"), stranded); + assert_eq!(rev(clone, "origin/main"), origin_main); + let upstream = run_git_in( + clone, + &[ + "for-each-ref", + "--format=%(upstream)", + "refs/heads/feature-new", + ], + ); + assert_eq!(upstream.trim(), ""); +} + +/// Work in progress on the base branch belongs with the commits beside it. +#[test] +fn uncommitted_changes_beside_stranded_commits_come_along_too() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let stranded = commit(clone, "one.txt", "agent work"); + std::fs::write(clone.join("file.txt"), "base\nin progress\n").unwrap(); + std::fs::write(clone.join("new.txt"), "untracked\n").unwrap(); + let before = uncommitted(clone); + assert_eq!(before.len(), 2); + + check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + + assert_eq!(head_branch(clone), "feature-new"); + assert_eq!(rev(clone, "HEAD"), stranded); + assert_eq!(uncommitted(clone), before); +} + +/// Once the commits have a branch of their own they are not stranded, and +/// the next branch of the clone starts from origin's base as any new branch +/// does, whether the clone was left on the first branch or back on `main`. +#[test] +fn a_later_branch_starts_from_origins_base_not_from_the_brought_along_commits() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let origin_main = rev(clone, "origin/main"); + let stranded = commit(clone, "one.txt", "agent work"); + check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + + check_out(clone, &BranchCheckout::new("main", "feature-second")).unwrap(); + assert_eq!(head_branch(clone), "feature-second"); + assert_eq!(rev(clone, "HEAD"), origin_main); + + run_git_in(clone, &["checkout", "main"]); + check_out(clone, &BranchCheckout::new("main", "feature-third")).unwrap(); + assert_eq!(head_branch(clone), "feature-third"); + assert_eq!(rev(clone, "HEAD"), origin_main); + + assert_eq!(rev(clone, "feature-new"), stranded); + assert_eq!(rev(clone, "main"), stranded); +} + +/// The clone already has the branch, so the commits on `main` can't be +/// brought along by creating it. Switching would leave them behind: the +/// checkout is refused with both branches and the commits named, and the +/// clone is untouched. +#[test] +fn stranded_commits_are_not_left_behind_for_a_branch_the_clone_has() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + run_git_in(clone, &["branch", "feature-x", "origin/main"]); + let feature = rev(clone, "feature-x"); + for n in 1..=7 { + commit(clone, &format!("{n}.txt"), &format!("agent work {n}")); + } + let stranded = rev(clone, "HEAD"); + + let error = refusal(check_out(clone, &BranchCheckout::new("main", "feature-x"))); + + assert!(error.contains("the clone is on 'main'"), "{error}"); + assert!(error.contains("7 commit(s)"), "{error}"); + assert!( + error.contains("the clone has a branch 'feature-x'"), + "{error}" + ); + // Newest first, five of them, then a count. + assert!(error.contains("agent work 7"), "{error}"); + assert!(error.contains("agent work 3"), "{error}"); + assert!(!error.contains("agent work 2"), "{error}"); + assert!(error.contains("and 2 more"), "{error}"); + assert_eq!(head_branch(clone), "main"); + assert_eq!(rev(clone, "main"), stranded); + assert_eq!(rev(clone, "feature-x"), feature); +} + +/// The same when the branch is only on origin (a pull request's branch). +#[test] +fn stranded_commits_are_not_left_behind_for_a_branch_origin_has() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + run_git_in(clone, &["push", "origin", "HEAD:refs/heads/feature-pr"]); + let stranded = commit(clone, "one.txt", "agent work"); + + let error = refusal(check_out(clone, &BranchCheckout::new("main", "feature-pr"))); + + assert!(error.contains("the clone is on 'main'"), "{error}"); + assert!( + error.contains("origin has a branch 'feature-pr'"), + "{error}" + ); + assert!(error.contains("agent work"), "{error}"); + assert_eq!(head_branch(clone), "main"); + assert_eq!(rev(clone, "main"), stranded); + assert!(!has_branch(clone, "feature-pr")); +} + +/// A base branch that is itself a branch Staged works on in this clone (a +/// stacked branch's parent) holds that branch's own commits. They are not +/// brought along, and the new branch starts from origin's copy of the base. +/// Uncommitted changes there are the parent's work in progress and keep the +/// clone where it is. +#[test] +fn commits_on_a_base_branch_that_is_worked_on_stay_on_it() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let origin_main = rev(clone, "origin/main"); + let parents_own = commit(clone, "one.txt", "the parent's own work"); + let checkout = BranchCheckout { + base_ref: "main", + branch_name: "feature-child", + base_is_worked_on: true, + }; + + std::fs::write(clone.join("file.txt"), "base\nin progress\n").unwrap(); + let error = refusal(check_out(clone, &checkout)); + assert!(error.contains("branch 'main'"), "{error}"); + assert!( + error.contains("uncommitted changes to 1 file(s)"), + "{error}" + ); + assert_eq!(head_branch(clone), "main"); + assert!(!has_branch(clone, "feature-child")); + + run_git_in(clone, &["checkout", "--", "file.txt"]); + check_out(clone, &checkout).unwrap(); + assert_eq!(head_branch(clone), "feature-child"); + assert_eq!(rev(clone, "HEAD"), origin_main); + assert_eq!(rev(clone, "main"), parents_own); +} + +// ── Other states the clone can't leave ────────────────────────────────────── + +/// Uncommitted changes on another branch are that branch's work. Git would +/// carry them onto this one, so the checkout is refused and the clone, its +/// changes included, is as it was. +#[test] +fn a_dirty_tree_on_another_branch_is_not_carried_across() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + run_git_in(clone, &["checkout", "-b", "feature-a"]); + std::fs::write(clone.join("file.txt"), "base\nin progress\n").unwrap(); + let before = uncommitted(clone); + + let error = refusal(check_out(clone, &BranchCheckout::new("main", "feature-b"))); + + assert!(error.contains("branch 'feature-a'"), "{error}"); + assert!(error.contains("'feature-b'"), "{error}"); + assert!( + error.contains("uncommitted changes to 1 file(s)"), + "{error}" + ); + assert_eq!(head_branch(clone), "feature-a"); + assert_eq!(uncommitted(clone), before); + assert!(!has_branch(clone, "feature-b")); +} + +/// A detached HEAD holding commits no ref reaches would lose them. One that +/// sits on a commit some ref has loses nothing and is left freely. +#[test] +fn a_detached_head_is_left_only_when_no_commit_is_lost() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + run_git_in(clone, &["checkout", "--detach"]); + let orphan = commit(clone, "one.txt", "work on no branch"); + + let error = refusal(check_out( + clone, + &BranchCheckout::new("main", "feature-new"), + )); + + assert!(error.contains("a detached HEAD"), "{error}"); + assert!(error.contains("work on no branch"), "{error}"); + assert!(error.contains("'feature-new'"), "{error}"); + assert_eq!(head_branch(clone), "HEAD"); + assert_eq!(rev(clone, "HEAD"), orphan); + assert!(!has_branch(clone, "feature-new")); + + run_git_in(clone, &["checkout", "--detach", "origin/main"]); + check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + assert_eq!(head_branch(clone), "feature-new"); +} + +// ── Through setup ─────────────────────────────────────────────────────────── + +/// End to end: a branch whose agent committed on `main` is set up with those +/// commits on it and marked `Running`. +#[tokio::test] +async fn setup_brings_the_commits_an_agent_left_on_the_base_branch() { + let (_origin, clone) = origin_and_workstation_clone(); + let stranded = commit(clone.path(), "one.txt", "agent work"); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(rev(clone.path(), "HEAD"), stranded); + assert_eq!(rev(clone.path(), "main"), stranded); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A branch that was never `Running` and can't be checked out without +/// leaving commits behind fails where the user can see it, with the reason +/// that names the branches and the commits. +#[tokio::test] +async fn setup_refused_over_stranded_commits_fails_with_the_reason() { + let (_origin, clone) = origin_and_workstation_clone(); + run_git_in(clone.path(), &["branch", "feature-new", "origin/main"]); + let stranded = commit(clone.path(), "one.txt", "agent work"); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + let error = set_up(&store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + for part in ["'main'", "'feature-new'", "agent work"] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(rev(clone.path(), "main"), stranded); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.failure_of(&branch), Some(error)); + assert_eq!(host.retries().len(), 1); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/poll.rs b/apps/staged/src-tauri/src/workspace_setup/tests/poll.rs new file mode 100644 index 000000000..84b5c392f --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/poll.rs @@ -0,0 +1,360 @@ +//! The status pollers: what a poll writes, and what it asks setup for +//! instead of writing `Running` itself. + +use std::sync::{Arc, Mutex}; + +use super::super::poll::{self, PollEffects, Reported}; +use super::super::SetupTrigger; +use super::{ + head_branch, origin_and_workstation_clone, set_status, set_up, starting_branch, status_of, + store_with_project, FakeHost, Info, Start, WORKSPACE, +}; +use crate::blox::{BloxError, WorkspaceListEntry}; +use crate::store::{self, Store, WorkspaceStatus}; + +/// What a poll did besides writing the row. +#[derive(Default)] +struct Seen { + written: Mutex>, + triggered: Mutex>, + retry_armed: Mutex>, + failed: Mutex>, +} + +impl Seen { + fn effects(&self) -> PollEffects<'_> { + PollEffects { + status_written: Box::new(|branch_id, changed, status| { + self.written + .lock() + .unwrap() + .push((branch_id.to_string(), changed, status.clone())); + }), + trigger_setup: Box::new(|branch_id| { + self.triggered.lock().unwrap().push(branch_id.to_string()); + }), + arm_retry: Box::new(|branch_id| { + self.retry_armed.lock().unwrap().push(branch_id.to_string()); + }), + failed: Box::new(|branch_id, reason| { + self.failed + .lock() + .unwrap() + .push((branch_id.to_string(), reason.to_string())); + }), + } + } + + fn retry_armed(&self) -> Vec { + self.retry_armed.lock().unwrap().clone() + } + + fn failed(&self) -> Vec<(String, String)> { + self.failed.lock().unwrap().clone() + } + + fn triggered(&self) -> Vec { + self.triggered.lock().unwrap().clone() + } + + fn written(&self) -> Vec<(String, bool, WorkspaceStatus)> { + self.written.lock().unwrap().clone() + } +} + +fn listed(status: &str) -> Vec { + vec![WorkspaceListEntry { + name: WORKSPACE.to_string(), + workstation_id: Some(42), + status: Some(status.to_string()), + extra: serde_json::Value::Null, + }] +} + +fn poll_batch(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { + let settled = poll::settle_all( + store, + &[branch.id.clone()], + &listed(blox_status), + &seen.effects(), + ); + settled.results[&branch.id].status.clone() +} + +fn poll_single(store: &Store, branch: &store::Branch, blox_status: &str, seen: &Seen) -> String { + poll::settle_one( + store, + branch, + WORKSPACE, + Ok(Reported { + status: Some(blox_status), + workstation_id: Some(42), + }), + &seen.effects(), + ) + .unwrap() + .status +} + +/// The other half of the bug: a poll saw the workstation running and marked +/// the branch `Running` with nothing checked out. Both pollers now leave the +/// row in `Starting` and ask for setup. +#[test] +fn a_poll_triggers_setup_instead_of_promoting_a_starting_branch() { + for poll in [poll_batch, poll_single] { + for blox_status in ["running", "ready", "active"] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let seen = Seen::default(); + + let reported = poll(&store, &branch, blox_status, &seen); + + assert_eq!(reported, "starting"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); + assert_eq!(seen.triggered(), vec![branch.id.clone()]); + // Nothing was written, so no session queue was kicked. + assert_eq!(seen.written(), vec![]); + } + } +} + +/// A batch poll also keeps fetching bootstrap progress for such a branch. +#[test] +fn a_batch_poll_keeps_a_branch_awaiting_setup_among_the_starting() { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let seen = Seen::default(); + + let settled = poll::settle_all( + &store, + &[branch.id.clone()], + &listed("running"), + &seen.effects(), + ); + + assert_eq!( + settled.starting_workspaces.get(WORKSPACE), + Some(&vec![branch.id.clone()]) + ); +} + +/// A setup that failed left the row in `Error`. A client still polling on a +/// stale `starting` is told so, rather than handed a `Running` branch whose +/// checkout never happened. The workstation being up is what a retry waits +/// for, so the poll makes sure the branch has its timer. +#[test] +fn a_poll_does_not_clear_a_failed_setup() { + for poll in [poll_batch, poll_single] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Error); + let seen = Seen::default(); + + let reported = poll(&store, &branch, "running", &seen); + + assert_eq!(reported, "error"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(seen.triggered(), Vec::::new()); + assert_eq!(seen.retry_armed(), vec![branch.id.clone()]); + assert_eq!(seen.written(), vec![]); + } +} + +/// A workstation that is running behind a row that says `Stopped`, +/// `Suspended` or nothing was resumed outside Staged, or by a poll that had +/// moved a `Starting` row to `Suspended` on the way. Nothing has read its +/// clone, so neither poller says `Running`: the row goes to `Starting` and +/// setup is asked for, the same road a new branch takes. +#[test] +fn a_poll_sends_a_branch_whose_workstation_came_back_through_setup() { + use WorkspaceStatus::*; + for poll in [poll_batch, poll_single] { + for row in [Some(Suspended), Some(Stopped), None] { + let (store, project) = store_with_project(); + let mut branch = + store::Branch::new_remote(&project.id, "feature-new", "main", WORKSPACE); + branch.workspace_status = row.clone(); + store.create_branch(&branch).unwrap(); + let branch = store.get_branch(&branch.id).unwrap().unwrap(); + assert_eq!(branch.workspace_status, row); + let seen = Seen::default(); + + let reported = poll(&store, &branch, "running", &seen); + + assert_eq!(reported, "starting", "{row:?}"); + assert_eq!(status_of(&store, &branch), Some(Starting), "{row:?}"); + assert_eq!(seen.triggered(), vec![branch.id.clone()], "{row:?}"); + // A `Starting` write kicks no session queue. + assert_eq!(seen.written(), vec![(branch.id.clone(), true, Starting)]); + } + } +} + +/// The two-poll route to an unverified `Running`: a `Starting` branch whose +/// workstation is seen suspended, then running. The second poll puts it back +/// in `Starting` and asks for setup, which is what ends in `Running`. +#[tokio::test] +async fn a_starting_branch_seen_suspended_then_running_is_still_set_up() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + let seen = Seen::default(); + + assert_eq!(poll_batch(&store, &branch, "suspended", &seen), "suspended"); + let branch = store.get_branch(&branch.id).unwrap().unwrap(); + assert_eq!(poll_batch(&store, &branch, "running", &seen), "starting"); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(seen.triggered(), vec![branch.id.clone()]); + + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// A poll that moves a row to `Error` itself says why, so the branch has a +/// reason to show and a retry timer like one setup failed. +#[test] +fn a_poll_that_fails_a_branch_says_why() { + for poll in [poll_batch, poll_single] { + for (row, blox_status) in [ + (WorkspaceStatus::Running, "degraded"), + (WorkspaceStatus::Starting, "failed"), + ] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, row); + let seen = Seen::default(); + + assert_eq!(poll(&store, &branch, blox_status, &seen), "error"); + + let failed = seen.failed(); + assert_eq!(failed.len(), 1, "{failed:?}"); + assert_eq!(failed[0].0, branch.id); + assert!(failed[0].1.contains(blox_status), "{failed:?}"); + assert!(failed[0].1.contains(WORKSPACE), "{failed:?}"); + + // Seen failed again, the reason it already has is kept. + let branch = store.get_branch(&branch.id).unwrap().unwrap(); + assert_eq!(poll(&store, &branch, blox_status, &seen), "error"); + assert_eq!(seen.failed().len(), 1); + } + } + + // A signed-out `sq` ends a single poll in `Error` too. + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let seen = Seen::default(); + let error = poll::settle_one( + &store, + &branch, + WORKSPACE, + Err(&BloxError::NotAuthenticated), + &seen.effects(), + ) + .err() + .unwrap(); + assert_eq!(error, "Not authenticated with Blox. Run: sq login"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(seen.failed(), vec![(branch.id.clone(), error)]); +} + +/// Every transition that does not end in `Running` is still the pollers', +/// and so is confirming a row setup already marked `Running`. +#[test] +fn a_poll_still_owns_the_other_transitions() { + use WorkspaceStatus::*; + for poll in [poll_batch, poll_single] { + for (row, blox_status, expected) in [ + (Running, "suspended", Suspended), + (Running, "stopped", Stopped), + (Running, "degraded", Error), + (Running, "running", Running), + (Starting, "suspended", Suspended), + (Starting, "failed", Error), + (Starting, "provisioning", Starting), + // Blox briefly reports a new workstation as stopped. + (Starting, "stopped", Starting), + (Suspended, "stopped", Stopped), + (Stopped, "suspended", Suspended), + ] { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, row.clone()); + let seen = Seen::default(); + + let reported = poll(&store, &branch, blox_status, &seen); + + assert_eq!(reported, expected.as_str(), "{row:?} + {blox_status}"); + assert_eq!(status_of(&store, &branch), Some(expected.clone())); + assert_eq!(seen.triggered(), Vec::::new()); + let changed = row != expected; + assert_eq!( + seen.written(), + vec![(branch.id.clone(), changed, expected)], + "{row:?} + {blox_status}" + ); + } + } +} + +/// A repo joining a workstation a peer already runs on is still held at +/// `Starting` by the pollers, and left to the setup that is cloning it. +#[test] +fn a_secondary_clone_is_still_held_at_starting() { + let (store, project) = store_with_project(); + let peer = starting_branch(&store, &project, "feature-one"); + set_status(&store, &peer, WorkspaceStatus::Running); + let repo = store::ProjectRepo::new(&project.id, "acme/gadgets", "feature-two", None); + store.create_project_repo(&repo).unwrap(); + let secondary = store::Branch::new_remote(&project.id, "feature-two", "main", WORKSPACE) + .with_project_repo(&repo.id); + store.create_branch(&secondary).unwrap(); + let seen = Seen::default(); + + assert_eq!(poll::is_secondary_clone_setup(&store, &secondary), Ok(true)); + assert_eq!(poll_batch(&store, &secondary, "running", &seen), "starting"); + + assert_eq!( + status_of(&store, &secondary), + Some(WorkspaceStatus::Starting) + ); + assert_eq!(seen.triggered(), Vec::::new()); + assert_eq!(seen.written(), vec![]); +} + +/// The poll's trigger, followed through: the branch a poll refused to +/// promote ends up checked out and `Running` by setup's hand. +#[tokio::test] +async fn a_polled_starting_branch_is_set_up_and_then_reported_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + let seen = Seen::default(); + + assert_eq!(poll_batch(&store, &branch, "running", &seen), "starting"); + for branch_id in seen.triggered() { + assert_eq!(branch_id, branch.id); + set_up(&store, &host, &branch, SetupTrigger::Observed) + .await + .unwrap(); + } + + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(head_branch(clone.path()), "feature-new"); + let branch = store.get_branch(&branch.id).unwrap().unwrap(); + assert_eq!(poll_batch(&store, &branch, "running", &seen), "running"); + assert_eq!(seen.triggered().len(), 1); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs b/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs new file mode 100644 index 000000000..3e49c16fc --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs @@ -0,0 +1,231 @@ +//! What a caller of setup gets back: an answer to what it asked, whatever +//! run it happened to land on, and without being held for a wait that is +//! not its own. + +use std::sync::Arc; +use std::time::Duration; + +use super::super::guard::{Patience, SetupGuard}; +use super::super::host::{ReadyWait, SetupHost}; +use super::super::{run_guarded, SetupRequest, SetupTrigger}; +use super::verify::{running_branch, running_host}; +use super::{ + eventually, head_branch, origin_and_workstation_clone, set_status, set_up, set_up_under, + starting_branch, status_of, store_with_project, FakeHost, Info, Start, +}; +use crate::store::{self, Store, WorkspaceStatus}; + +/// A request as the `start_workspace` command makes it: it stays while the +/// run is working and leaves when the run is only waiting. +async fn request( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, +) -> Result, String> { + let host: Arc = host.clone(); + let request = SetupRequest::new(SetupTrigger::Requested); + run_guarded( + guard, + store, + &host, + &branch.id, + request, + Patience::WhileWorking, + ) + .await +} + +/// A run in a gated checkout, holding its workstation's turn. +async fn hold_the_workstation( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, +) -> tokio::task::JoinHandle, String>> { + let holder = { + let (guard, store, host, branch) = ( + Arc::clone(guard), + Arc::clone(store), + Arc::clone(host), + branch.clone(), + ); + tokio::spawn(async move { + set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested).await + }) + }; + host.wait_for_checkouts(1).await; + holder +} + +/// A resolved request used to mean nothing about the branch: a `Requested` +/// call that joined a poll's run got that run's `Ok`, and when the run had +/// found nothing to do (a poll had moved the row on while it queued) the +/// branch was left as it was with the caller told all was well. The caller +/// now looks again and runs its own setup. +#[tokio::test] +async fn a_request_that_joins_a_run_with_nothing_to_do_still_gets_its_setup() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let first = starting_branch(&store, &project, "feature-a"); + let branch = starting_branch(&store, &project, "feature-b"); + let (host, gate) = + FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds).gated(); + let host = Arc::new(host); + let guard = Arc::new(SetupGuard::default()); + + // Another branch's run has the workstation, so the poll's run for this + // branch queues behind it. + let holder = hold_the_workstation(&guard, &store, &host, &first).await; + let observed = { + let (guard, store, host, branch) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&host), + branch.clone(), + ); + tokio::spawn(async move { + set_up_under(&guard, &store, &host, &branch, SetupTrigger::Observed).await + }) + }; + eventually("the observed run is in flight", || { + guard.is_in_flight(&branch.id) + }) + .await; + // A poll sees the workstation suspended: the observed run, when its + // turn comes, has nothing to do. + set_status(&store, &branch, WorkspaceStatus::Suspended); + + // The request joins that run on its first poll, before the gate opens. + let (requested, ()) = tokio::join!( + set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested), + async { + tokio::task::yield_now().await; + gate.add_permits(8); + } + ); + + assert_eq!(requested, Ok(Some(WorkspaceStatus::Running))); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(head_branch(clone.path()), "feature-b"); + // The observed caller's own run did nothing, which is its answer. + assert!(observed.await.unwrap().is_ok()); + assert_eq!(holder.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); + assert_eq!(host.count("check_out"), 2); +} + +/// A workstation that is still booting can take minutes. The request is +/// answered `Starting` as soon as the run starts waiting, so it does not +/// hold one of the frontend's setup slots; the run goes on and ends in a +/// checked-out `Running` branch. +#[tokio::test] +async fn a_request_does_not_wait_out_a_booting_workstation() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let mut host = FakeHost::new( + clone.path(), + &[Info::Unreachable], + Start::Fails("timed out"), + ); + // Far longer than the test: the request must not be what waits it out. + host.ready_wait = ReadyWait { + timeout: Duration::from_secs(600), + interval: Duration::from_millis(5), + }; + let host = Arc::new(host); + let guard = Arc::new(SetupGuard::default()); + + let answered = request(&guard, &store, &host, &branch).await; + + assert_eq!(answered, Ok(Some(WorkspaceStatus::Starting))); + assert!(guard.is_in_flight(&branch.id)); + assert_eq!(host.count("start_workspace"), 1); + assert_eq!(host.count("check_out"), 0); + + // The workstation arrives; nobody is waiting on the run. + *host.info.lock().unwrap() = [Info::Status("running")].into_iter().collect(); + eventually("the run finishes on its own", || { + status_of(&store, &branch) == Some(WorkspaceStatus::Running) + }) + .await; + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(host.count("start_workspace"), 1); +} + +/// The same for a run that is behind another branch's on the workstation, +/// which may itself be waiting for it: the request is answered `Starting` +/// at once, whatever the row said (a failed branch's Retry), and the run +/// takes its turn. +#[tokio::test] +async fn a_request_queued_behind_another_branch_is_answered_at_once() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let first = starting_branch(&store, &project, "feature-a"); + let branch = starting_branch(&store, &project, "feature-b"); + let branch = set_status(&store, &branch, WorkspaceStatus::Error); + let (host, gate) = + FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds).gated(); + let host = Arc::new(host); + let guard = Arc::new(SetupGuard::default()); + let holder = hold_the_workstation(&guard, &store, &host, &first).await; + + let answered = request(&guard, &store, &host, &branch).await; + + assert_eq!(answered, Ok(Some(WorkspaceStatus::Starting))); + assert!(guard.is_in_flight(&branch.id)); + assert_eq!(host.count("check_out"), 1); + // The row agrees with the answer, so a poll of it does too. + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); + + gate.add_permits(8); + assert_eq!(holder.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); + eventually("the queued run takes its turn", || { + status_of(&store, &branch) == Some(WorkspaceStatus::Running) + }) + .await; + assert_eq!(head_branch(clone.path()), "feature-b"); +} + +/// A request that can be answered while the run works is: `Running`, with +/// the branch checked out. +#[tokio::test] +async fn a_request_for_a_running_workstation_is_answered_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = Arc::new(running_host(clone.path())); + let guard = Arc::new(SetupGuard::default()); + + let answered = request(&guard, &store, &host, &branch).await; + + assert_eq!(answered, Ok(Some(WorkspaceStatus::Running))); + assert_eq!(head_branch(clone.path()), "feature-new"); +} + +/// A request for a branch that is already `Running` is told so and starts +/// nothing. Any other trigger with nothing to do is told what the row says. +#[tokio::test] +async fn a_caller_with_nothing_to_do_is_told_what_the_row_says() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let running = running_branch(&store, &project, "feature-running"); + let failed = starting_branch(&store, &project, "feature-failed"); + let failed = set_status(&store, &failed, WorkspaceStatus::Error); + let host = Arc::new(running_host(clone.path())); + + assert_eq!( + set_up(&store, &host, &running, SetupTrigger::Requested).await, + Ok(Some(WorkspaceStatus::Running)) + ); + assert_eq!( + set_up(&store, &host, &failed, SetupTrigger::Observed).await, + Ok(Some(WorkspaceStatus::Error)) + ); + assert_eq!( + set_up(&store, &host, &running, SetupTrigger::Retry).await, + Ok(Some(WorkspaceStatus::Running)) + ); + assert_eq!(host.calls.lock().unwrap().len(), 0); + assert_eq!(head_branch(clone.path()), "main"); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs new file mode 100644 index 000000000..fec302a50 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs @@ -0,0 +1,331 @@ +//! `Error` has a way out: every failure records why and sets a retry, the +//! retry sets the branch up once its workstation is running, and a refused +//! repair does not turn a branch that was `Running` into a failed one. + +use std::sync::Arc; +use std::time::Duration; + +use super::super::guard::SetupGuard; +use super::super::host::SetupHost; +use super::super::retry::{self, RetryPolicy}; +use super::super::SetupTrigger; +use super::verify::{running_branch, running_host, start_session_on}; +use super::{ + head_branch, origin_and_workstation_clone, set_status, set_up_under, starting_branch, + status_of, store_with_project, FakeHost, Info, Start, +}; +use crate::store::{self, Store, WorkspaceStatus}; +use crate::test_utils::run_git_in; + +const MINUTE: Duration = Duration::from_secs(60); + +/// A host whose checkout fails the way a dropped exec does. +fn failing_host(clone: &std::path::Path) -> Arc { + let mut host = running_host(clone); + host.checkout_error = Some("could not fetch base branch 'main': connection reset"); + Arc::new(host) +} + +fn host_reporting(clone: &std::path::Path, info: Info) -> Arc { + Arc::new(FakeHost::new(clone, &[info], Start::Succeeds)) +} + +/// Fail `branch`'s setup once and return the ticket of the timer that set. +async fn fail_once( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, +) -> u64 { + set_up_under(guard, store, host, branch, SetupTrigger::Requested) + .await + .unwrap_err(); + host.retries().last().unwrap().2 +} + +async fn fire( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, + ticket: u64, +) { + let host: Arc = host.clone(); + retry::fire(guard, store, &host, &branch.id, ticket).await; +} + +fn calls(host: &FakeHost) -> Vec<&'static str> { + host.calls.lock().unwrap().clone() +} + +#[test] +fn the_wait_doubles_from_a_minute_to_half_an_hour() { + let policy = RetryPolicy::DEFAULT; + let waits: Vec = (0..7).map(|n| policy.delay(n).as_secs()).collect(); + assert_eq!(waits, vec![60, 120, 240, 480, 960, 1800, 1800]); + assert_eq!(policy.delay(u32::MAX), Duration::from_secs(1800)); +} + +/// A failed setup leaves the row in `Error` with its reason announced and a +/// retry set, and each further failure waits twice as long. +#[tokio::test] +async fn a_failed_setup_says_why_and_sets_a_retry() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let host = failing_host(clone.path()); + let guard = Arc::new(SetupGuard::default()); + + let error = set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested) + .await + .unwrap_err(); + + assert!(error.contains("connection reset"), "{error}"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!( + host.failures(), + vec![ + // Setting out clears what the last attempt left. + (branch.id.clone(), None), + (branch.id.clone(), Some(error)), + ] + ); + assert_eq!(host.retries(), vec![(branch.id.clone(), MINUTE, 1)]); + assert!(guard.retry_is_armed(&branch.id)); + + fail_once(&guard, &store, &host, &branch).await; + assert_eq!(host.retries()[1], (branch.id.clone(), 2 * MINUTE, 2)); +} + +/// The timer fires with the workstation running: the branch is set up +/// without anyone asking, its reason is cleared and its schedule forgotten. +/// The workstation is never started for it. +#[tokio::test] +async fn a_retry_sets_the_branch_up_when_its_workstation_is_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let ticket = fail_once(&guard, &store, &failing_host(clone.path()), &branch).await; + let host = Arc::new(running_host(clone.path())); + + fire(&guard, &store, &host, &branch, ticket).await; + + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(head_branch(clone.path()), "feature-new"); + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(host.failures(), vec![(branch.id.clone(), None)]); + assert_eq!( + host.written(), + vec![(branch.id.clone(), true, WorkspaceStatus::Running)] + ); + assert_eq!(host.retries(), vec![]); + assert!(!guard.retry_is_armed(&branch.id)); +} + +/// A retry is not what starts or resumes a workstation. One that is down, +/// or can't be asked about, leaves the branch in `Error` with its reason, +/// touches nothing and looks again later. +#[tokio::test] +async fn a_retry_never_starts_a_workstation() { + for info in [ + Info::Status("stopped"), + Info::Status("suspended"), + Info::Unreachable, + ] { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let ticket = fail_once(&guard, &store, &failing_host(clone.path()), &branch).await; + let host = host_reporting(clone.path(), info); + + fire(&guard, &store, &host, &branch, ticket).await; + + assert_eq!(calls(&host), vec!["workspace_status"]); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert_eq!(host.failures(), vec![]); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(host.retries(), vec![(branch.id.clone(), 2 * MINUTE, 2)]); + assert!(guard.retry_is_armed(&branch.id)); + } +} + +/// A branch has one timer that counts. An older one, superseded by a later +/// failure, fires into nothing. +#[tokio::test] +async fn only_the_latest_timer_counts() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let failing = failing_host(clone.path()); + let first = fail_once(&guard, &store, &failing, &branch).await; + let second = fail_once(&guard, &store, &failing, &branch).await; + let host = Arc::new(running_host(clone.path())); + + fire(&guard, &store, &host, &branch, first).await; + assert_eq!(calls(&host), Vec::<&str>::new()); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + + fire(&guard, &store, &host, &branch, second).await; + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + + // And one that has fired does not fire twice. + let before = calls(&host); + fire(&guard, &store, &host, &branch, second).await; + assert_eq!(calls(&host), before); +} + +/// A timer for a branch that is no longer `Error` (a poll saw it suspended, +/// the user's Retry got there first) does nothing and its schedule is +/// dropped, so the next failure starts from a minute again. +#[tokio::test] +async fn a_timer_for_a_branch_that_moved_on_does_nothing() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let failing = failing_host(clone.path()); + let ticket = fail_once(&guard, &store, &failing, &branch).await; + set_status(&store, &branch, WorkspaceStatus::Suspended); + let host = Arc::new(running_host(clone.path())); + + fire(&guard, &store, &host, &branch, ticket).await; + + assert_eq!(calls(&host), Vec::<&str>::new()); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Suspended)); + assert!(!guard.retry_is_armed(&branch.id)); + + fail_once(&guard, &store, &failing, &branch).await; + assert_eq!(failing.retries().last().unwrap().1, MINUTE); +} + +/// A setup that works, however it was asked for, ends the schedule: the +/// timer still out for the branch finds nothing to do. +#[tokio::test] +async fn a_setup_that_works_forgets_the_schedule() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let ticket = fail_once(&guard, &store, &failing_host(clone.path()), &branch).await; + let host = Arc::new(running_host(clone.path())); + + // The error view's Retry. + let status = set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested).await; + + assert_eq!(status, Ok(Some(WorkspaceStatus::Running))); + assert_eq!(host.failures(), vec![(branch.id.clone(), None)]); + assert!(!guard.retry_is_armed(&branch.id)); + let before = calls(&host); + fire(&guard, &store, &host, &branch, ticket).await; + assert_eq!(calls(&host), before); +} + +/// What finds a branch in `Error` without having failed it (a poll, the +/// startup pass) sets its timer once, however often it looks. +#[tokio::test] +async fn a_branch_seen_failed_gets_one_timer() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let branch = set_status(&store, &branch, WorkspaceStatus::Error); + let guard = Arc::new(SetupGuard::default()); + let host = host_reporting(clone.path(), Info::Status("stopped")); + + for _ in 0..3 { + retry::arm(&guard, host.as_ref(), &branch.id); + } + assert_eq!(host.retries(), vec![(branch.id.clone(), MINUTE, 1)]); + + // Fired and set again by the retry itself: still one. + fire(&guard, &store, &host, &branch, 1).await; + retry::arm(&guard, host.as_ref(), &branch.id); + assert_eq!(host.retries().len(), 2); +} + +/// A repair that is refused changed nothing in the clone, so a branch that +/// was `Running` is `Running` again: only the session that asked fails, +/// with the reason. Here the clone is on another branch with uncommitted +/// changes, which a checkout would carry across. +#[tokio::test] +async fn a_refused_repair_leaves_a_running_branch_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + std::fs::write(clone.path().join("file.txt"), "base\nin progress\n").unwrap(); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + for part in [ + "had branch 'feature-a' checked out", + "this session's branch 'feature-b'", + "uncommitted changes to 1 file(s)", + ] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(host.failure_of(&branch), None); + assert_eq!(host.retries(), vec![]); + // The row went through `Starting`; the write back kicks the queue that + // held meanwhile. + assert_eq!( + host.written(), + vec![(branch.id.clone(), true, WorkspaceStatus::Running)] + ); + assert_eq!(head_branch(clone.path()), "feature-a"); + let dirty = run_git_in(clone.path(), &["status", "--porcelain"]); + assert!(dirty.contains("file.txt"), "{dirty}"); +} + +/// The same for commits a switch would leave behind on the base branch. +#[tokio::test] +async fn a_repair_refused_over_stranded_commits_leaves_the_branch_running() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["branch", "feature-b", "origin/main"]); + let stranded = super::checkout::commit(clone.path(), "one.txt", "agent work"); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + for part in ["'main'", "'feature-b'", "agent work"] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(host.failure_of(&branch), None); + assert_eq!(host.retries(), vec![]); + assert_eq!(head_branch(clone.path()), "main"); + let main = run_git_in(clone.path(), &["rev-parse", "main"]); + assert_eq!(main.trim(), stranded); +} + +/// A repair that finds the workstation down is refused too: what the row of +/// a stopped workstation should say is the pollers' to settle. +#[tokio::test] +async fn a_repair_never_starts_a_workstation_for_a_running_row() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-b"); + // The clone reads as on another branch; Blox says the workstation is + // stopped by the time the repair looks. + let host = host_reporting(clone.path(), Info::Status("stopped")); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + assert!(error.contains("is not running"), "{error}"); + assert_eq!(host.count("start_workspace"), 0); + assert_eq!(host.count("check_out"), 0); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(host.retries(), vec![]); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/sessions.rs b/apps/staged/src-tauri/src/workspace_setup/tests/sessions.rs new file mode 100644 index 000000000..e52971a03 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/sessions.rs @@ -0,0 +1,338 @@ +//! The sessions sharing a clone: a checkout never lands between an agent +//! start's read of the clone and its launch, a `disconnected` row does not +//! hold a clone forever, and "the same clone" means the directory on the +//! workstation, whichever project a branch belongs to. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use super::super::clone::checkout_for; +use super::super::guard::SetupGuard; +use super::super::host::SetupHost; +use super::super::verify::verify_session_checkout; +use super::super::SetupTrigger; +use super::checkout::commit; +use super::verify::{live_session, running_branch, running_host, start_session_on}; +use super::{ + head_branch, origin_and_workstation_clone, set_status, set_up_under, starting_branch, + status_of, store_with_project, FakeHost, WORKSPACE, +}; +use crate::store::{self, SessionStatus, Store, WorkspaceStatus}; +use crate::test_utils::run_git_in; + +/// A session of `branch` whose proxy connection was lost: its turn may +/// still be going on the workstation. +fn disconnected_session(store: &Store, branch: &store::Branch) -> store::Session { + let session = live_session(store, branch); + store + .update_session_status(&session.id, SessionStatus::Disconnected, None, None) + .unwrap(); + session +} + +/// A host whose reconcile service, asked to look at a disconnected session, +/// finds what became of it and writes `settled` on the row. +fn host_settling(clone: &Path, store: &Arc, settled: SessionStatus) -> Arc { + let mut host = running_host(clone); + let store = Arc::clone(store); + host.on_check_disconnected = Some(Box::new(move |session_id| { + store + .update_session_status(session_id, settled.clone(), None, None) + .unwrap(); + })); + Arc::new(host) +} + +// ── The start's read and the launch are one step ──────────────────────────── + +/// A session start read the clone as on its branch and has not launched +/// yet: its row is not live, so nothing would stop a sibling branch's setup +/// from switching the clone under it. The start holds the clone until the +/// row is live. The setup that was waiting then finds a live session there +/// and leaves the clone alone. +#[tokio::test] +async fn a_checkout_does_not_land_between_a_starts_read_and_its_launch() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-a"); + let sibling = starting_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + let host = Arc::new(running_host(clone.path())); + let dyn_host: Arc = host.clone(); + let guard = Arc::new(SetupGuard::default()); + + // The start has read the clone and holds it. + let hold = verify_session_checkout(&guard, &store, &dyn_host, &branch, "session-a") + .await + .unwrap(); + + let setup = { + let (guard, store, host, sibling) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&host), + sibling.clone(), + ); + tokio::spawn(async move { + set_up_under(&guard, &store, &host, &sibling, SetupTrigger::Requested).await + }) + }; + // The setup gets as far as the clone and no further. + host.wait_for_calls("workspace_status", 1).await; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); + + // The session goes live, and only then lets go. + live_session(&store, &branch); + drop(hold); + + let error = setup.await.unwrap().unwrap_err(); + assert!(error.contains("branch 'feature-a'"), "{error}"); + assert!(error.contains("running in the same clone"), "{error}"); + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); +} + +/// The other order: a start does not read a clone that is mid-checkout. It +/// reads after, sees where the clone was left, and repairs it. +#[tokio::test] +async fn a_start_does_not_read_a_clone_that_is_mid_checkout() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-a"); + let sibling = starting_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + let (host, gate) = running_host(clone.path()).gated(); + let host = Arc::new(host); + let guard = Arc::new(SetupGuard::default()); + + let setup = { + let (guard, store, host, sibling) = ( + Arc::clone(&guard), + Arc::clone(&store), + Arc::clone(&host), + sibling.clone(), + ); + tokio::spawn(async move { + set_up_under(&guard, &store, &host, &sibling, SetupTrigger::Requested).await + }) + }; + host.wait_for_checkouts(1).await; + + let start = { + let (guard, store, branch) = (Arc::clone(&guard), Arc::clone(&store), branch.clone()); + let host: Arc = host.clone(); + tokio::spawn(async move { + verify_session_checkout(&guard, &store, &host, &branch, "session-a") + .await + .map(drop) + }) + }; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(host.count("clone_head"), 0, "read under a checkout"); + + gate.add_permits(8); + assert_eq!(setup.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); + start.await.unwrap().unwrap(); + assert_eq!(head_branch(clone.path()), "feature-a"); +} + +// ── Disconnected sessions ─────────────────────────────────────────────────── + +/// A `disconnected` row used to hold its clone until the reconcile service +/// got round to it, up to half an hour later. Setup now asks the service to +/// look at once; a turn that turns out to be over stops counting, and the +/// repair goes ahead. +#[tokio::test] +async fn a_disconnected_session_that_is_over_stops_holding_the_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let sibling = running_branch(&store, &project, "feature-a"); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + let lost = disconnected_session(&store, &sibling); + let host = host_settling(clone.path(), &store, SessionStatus::Completed); + + start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap(); + + assert_eq!(host.checked_sessions(), vec![lost.id]); + assert_eq!(head_branch(clone.path()), "feature-b"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +/// One whose turn is still going is claimed `running` again and holds the +/// clone for what it is. +#[tokio::test] +async fn a_disconnected_session_that_is_still_working_holds_the_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let sibling = running_branch(&store, &project, "feature-a"); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + disconnected_session(&store, &sibling); + let host = host_settling(clone.path(), &store, SessionStatus::Running); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + assert!(error.contains("is running in the same clone"), "{error}"); + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); +} + +/// One the service could not settle within the wait still counts as live, +/// and the refusal says what is being waited for. The branch that asked is +/// left `Running`. +#[tokio::test] +async fn a_disconnected_session_nobody_settled_holds_the_clone_and_says_so() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let sibling = running_branch(&store, &project, "feature-a"); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + let lost = disconnected_session(&store, &sibling); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + assert!(error.contains("branch 'feature-a'"), "{error}"); + assert!(error.contains("lost its connection"), "{error}"); + assert!(error.contains("Staged is checking on it"), "{error}"); + assert_eq!(host.checked_sessions(), vec![lost.id]); + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); +} + +// ── What "the same clone" covers ──────────────────────────────────────────── + +/// Two projects on one repository and one workstation share the clone. A +/// live session of the other project's branch holds it like any other. +#[tokio::test] +async fn a_live_session_of_another_project_holds_the_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let other_project = store::Project::named("widgets again").with_primary_repo("acme/widgets"); + store.create_project(&other_project).unwrap(); + let sibling = running_branch(&store, &other_project, "feature-a"); + let branch = running_branch(&store, &project, "feature-b"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + live_session(&store, &sibling); + let host = Arc::new(running_host(clone.path())); + + let error = start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap_err(); + + assert!(error.contains("branch 'feature-a'"), "{error}"); + assert!(error.contains("running in the same clone"), "{error}"); + assert_eq!(host.count("check_out"), 0); + assert_eq!(head_branch(clone.path()), "feature-a"); +} + +/// A session on another workstation is in another clone and holds nothing +/// here, even within the one project. +#[tokio::test] +async fn a_live_session_on_another_workstation_does_not_hold_the_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let elsewhere = + store::Branch::new_remote(&project.id, "feature-a", "main", "another-workstation"); + store.create_branch(&elsewhere).unwrap(); + let elsewhere = set_status(&store, &elsewhere, WorkspaceStatus::Running); + live_session(&store, &elsewhere); + let branch = running_branch(&store, &project, "feature-b"); + let host = Arc::new(running_host(clone.path())); + + start_session_on(&store, &host, &branch, "session-b") + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-b"); +} + +/// The session a repair was asked for is not what keeps its own clone from +/// being repaired (a resume's row is `running` before its checkout check). +#[tokio::test] +async fn the_session_being_started_does_not_hold_its_own_clone() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = running_branch(&store, &project, "feature-b"); + let own = live_session(&store, &branch); + let host = Arc::new(running_host(clone.path())); + + start_session_on(&store, &host, &branch, &own.id) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-b"); +} + +// ── A base branch that is worked on ───────────────────────────────────────── + +/// Whether the base branch is one Staged works on in this clone is read +/// from the rows: a remote branch of that name on the same workstation and +/// clone, in any project. +#[test] +fn the_base_branch_is_worked_on_when_a_branch_of_that_name_shares_the_clone() { + let (store, project) = store_with_project(); + let child = store::Branch::new_remote(&project.id, "feature-child", "feature-a", WORKSPACE); + store.create_branch(&child).unwrap(); + let worked_on = |store: &Arc| checkout_for(store, &child).unwrap().base_is_worked_on; + assert!(!worked_on(&store)); + + // Not a local branch of that name, nor one on another workstation. + store + .create_branch(&store::Branch::new(&project.id, "feature-a", "main")) + .unwrap(); + let elsewhere = + store::Branch::new_remote(&project.id, "feature-a", "main", "another-workstation"); + store.create_branch(&elsewhere).unwrap(); + assert!(!worked_on(&store)); + + let other_project = store::Project::named("widgets again").with_primary_repo("acme/widgets"); + store.create_project(&other_project).unwrap(); + starting_branch(&store, &other_project, "feature-a"); + assert!(worked_on(&store)); + + // A branch named like its own base is not what works on that base. + let (store, project) = store_with_project(); + let on_itself = store::Branch::new_remote(&project.id, "main", "main", WORKSPACE); + store.create_branch(&on_itself).unwrap(); + assert!(!checkout_for(&store, &on_itself).unwrap().base_is_worked_on); +} + +/// A stacked branch set up in a clone standing on its parent, which has +/// commits of its own that are not pushed: they stay the parent's, and the +/// child starts from origin's copy of the parent. +#[tokio::test] +async fn a_stacked_branch_does_not_take_its_parents_unpushed_commits() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + running_branch(&store, &project, "feature-a"); + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + commit(clone.path(), "pushed.txt", "pushed parent work"); + run_git_in(clone.path(), &["push", "origin", "feature-a"]); + let pushed = run_git_in(clone.path(), &["rev-parse", "origin/feature-a"]); + let unpushed = commit(clone.path(), "unpushed.txt", "unpushed parent work"); + let child = store::Branch::new_remote(&project.id, "feature-child", "feature-a", WORKSPACE); + store.create_branch(&child).unwrap(); + let host = Arc::new(running_host(clone.path())); + let guard = Arc::new(SetupGuard::default()); + + set_up_under(&guard, &store, &host, &child, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-child"); + assert_eq!(run_git_in(clone.path(), &["rev-parse", "HEAD"]), pushed); + let parent = run_git_in(clone.path(), &["rev-parse", "feature-a"]); + assert_eq!(parent.trim(), unpushed); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs index 5e65cfe9c..d67bfbf9b 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs @@ -5,36 +5,39 @@ use std::path::Path; use std::sync::{Arc, Mutex}; +use super::super::checkout::{local_branch_exists, BranchCheckout}; use super::super::guard::SetupGuard; +use super::super::host::SetupHost; use super::super::verify::{parse_clone_head, verify_session_checkout, CloneHead}; -use super::super::{run_guarded, SetupHost, SetupTrigger}; +use super::super::SetupTrigger; use super::{ - head_branch, origin_and_workstation_clone, probe_clone_head, set_status, set_up, + head_branch, origin_and_workstation_clone, probe_clone_head, set_status, set_up, set_up_under, starting_branch, status_of, store_with_project, FakeHost, Head, Info, Start, }; use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; use crate::test_utils::{run_git_in, TempDir}; -/// A branch whose row says `Running`, the way setup or a poller left it. -fn running_branch(store: &Store, project: &store::Project, name: &str) -> store::Branch { +/// A branch whose row says `Running`, the way setup left it. +pub(super) fn running_branch(store: &Store, project: &store::Project, name: &str) -> store::Branch { let branch = starting_branch(store, project, name); set_status(store, &branch, WorkspaceStatus::Running) } -fn running_host(clone: &Path) -> FakeHost { +pub(super) fn running_host(clone: &Path) -> FakeHost { FakeHost::new(clone, &[Info::Status("running")], Start::Succeeds) } /// A session row in flight on `branch`. -fn live_session(store: &Store, branch: &store::Branch) -> store::Session { +pub(super) fn live_session(store: &Store, branch: &store::Branch) -> store::Session { let session = store::Session::new_running("work", Path::new("/tmp")).with_branch(&branch.id); store.create_session(&session).unwrap(); session } -/// The session-start check, as `ensure_session_checkout` runs it. -async fn start_session_on( +/// The session-start check, as `ensure_session_checkout` runs it, with the +/// clone let go of as soon as it has answered. +pub(super) async fn start_session_on( store: &Arc, host: &Arc, branch: &store::Branch, @@ -42,7 +45,9 @@ async fn start_session_on( ) -> Result<(), String> { let guard = Arc::new(SetupGuard::default()); let host: Arc = host.clone(); - verify_session_checkout(&guard, store, &host, branch, session_id).await + verify_session_checkout(&guard, store, &host, branch, session_id) + .await + .map(drop) } fn calls(host: &FakeHost) -> Vec<&'static str> { @@ -208,8 +213,8 @@ async fn a_session_on_a_checked_out_branch_starts_after_one_read() { assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); } -/// The gap setup alone can't close: a branch that was never set up, resumed -/// outside Staged and promoted to `Running` by a poller, still on `main`. +/// The gap setup alone can't close: a `Running` branch whose clone has +/// since been left on `main` (a sibling branch's setup, a terminal). /// The session start checks the branch out through the shared setup and /// only then lets the agent start. The session being started is itself /// `running` on the branch by then and does not stand in the way. @@ -256,9 +261,11 @@ async fn a_session_start_repairs_a_missing_clone() { assert_eq!(head_branch(clone.path()), "feature-new"); } -/// When the repair fails the agent is not started: the session fails with -/// both branch names and the cause, and the branch stops saying `Running`, -/// so whatever is queued behind it waits instead of failing in turn. +/// When the repair fails for a reason that is not a refusal (here a fetch +/// that never reached origin) the agent is not started: the session fails +/// with both branch names and the cause, and the branch stops saying +/// `Running`, so whatever is queued behind it waits instead of failing in +/// turn. The reason is on show and a retry is set. #[tokio::test] async fn a_failed_repair_fails_the_session_and_the_branch() { let (_origin, clone) = origin_and_workstation_clone(); @@ -278,6 +285,9 @@ async fn a_failed_repair_fails_the_session_and_the_branch() { assert_eq!(head_branch(clone.path()), "main"); assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); assert_eq!(host.written(), vec![]); + let reason = host.failure_of(&branch).expect("a recorded reason"); + assert!(reason.contains("network unreachable"), "{reason}"); + assert_eq!(host.retries().len(), 1); } /// A read that doesn't answer is not "wrong branch" (no checkout runs) and @@ -430,7 +440,11 @@ async fn concurrent_session_starts_share_one_repair() { branch.clone(), ); tokio::spawn(async move { - verify_session_checkout(&guard, &store, &host, &branch, session_id).await + // Let go of at once: a start that kept the clone would + // hold the other one's read. + verify_session_checkout(&guard, &store, &host, &branch, session_id) + .await + .map(drop) }) }) .collect(); @@ -454,13 +468,13 @@ async fn a_repair_runs_the_shared_setup_under_the_guard() { let (store, project) = store_with_project(); let branch = running_branch(&store, &project, "feature-new"); let host = Arc::new(running_host(clone.path())); - let dyn_host: Arc = host.clone(); let guard = Arc::new(SetupGuard::default()); - run_guarded(&guard, &store, &dyn_host, &branch.id, SetupTrigger::Repair) + let status = set_up_under(&guard, &store, &host, &branch, SetupTrigger::Repair) .await .unwrap(); + assert_eq!(status, Some(WorkspaceStatus::Running)); assert_eq!(host.count("check_auth"), 1); assert_eq!(host.count("check_out"), 1); assert_eq!(head_branch(clone.path()), "feature-new"); @@ -497,8 +511,12 @@ fn a_failed_local_branch_probe_is_not_read_as_a_missing_branch() { real(args) }); - let error = branches::check_out_workspace_branch(&git, "main", "feature-x").unwrap_err(); + let error = + branches::check_out_workspace_branch(&git, &BranchCheckout::new("main", "feature-x")) + .unwrap_err(); + assert!(!error.is_refusal()); + let error = error.to_string(); assert!(error.contains("already has branch 'feature-x'"), "{error}"); assert!(error.contains("workstation disconnected"), "{error}"); let ran = ran.lock().unwrap(); @@ -520,21 +538,18 @@ fn the_local_branch_probe_answers_through_its_output() { let (_origin, clone, unpushed) = clone_with_an_unpushed_feature_branch(); let git = branches::git_runner(clone.path(), None, None); - assert_eq!(branches::local_branch_exists(&git, "feature-x"), Ok(true)); - assert_eq!(branches::local_branch_exists(&git, "feature-y"), Ok(false)); + assert_eq!(local_branch_exists(&git, "feature-x"), Ok(true)); + assert_eq!(local_branch_exists(&git, "feature-y"), Ok(false)); // A prefix of a branch name is not that branch. run_git_in(clone.path(), &["branch", "team/feature-z"]); - assert_eq!(branches::local_branch_exists(&git, "team"), Ok(false)); - assert_eq!( - branches::local_branch_exists(&git, "team/feature-z"), - Ok(true) - ); + assert_eq!(local_branch_exists(&git, "team"), Ok(false)); + assert_eq!(local_branch_exists(&git, "team/feature-z"), Ok(true)); - branches::check_out_workspace_branch(&git, "main", "feature-x").unwrap(); + branches::check_out_workspace_branch(&git, &BranchCheckout::new("main", "feature-x")).unwrap(); assert_eq!(head_branch(clone.path()), "feature-x"); assert_eq!(run_git_in(clone.path(), &["rev-parse", "HEAD"]), unpushed); - branches::check_out_workspace_branch(&git, "main", "feature-y").unwrap(); + branches::check_out_workspace_branch(&git, &BranchCheckout::new("main", "feature-y")).unwrap(); assert_eq!(head_branch(clone.path()), "feature-y"); assert_eq!( run_git_in(clone.path(), &["rev-parse", "HEAD"]), diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs new file mode 100644 index 000000000..ade9e0781 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -0,0 +1,493 @@ +//! That the triggers and checks are still called from where they should be. +//! +//! The behaviour of setup, the pollers' decisions and the session-start +//! check are tested against a faked workstation. What calls them is not: +//! the startup pass, the pollers' effects, the resume command and the +//! session and pipeline start paths all need a concrete app runtime to +//! drive. Deleting one of those calls would leave every behaviour test +//! passing and the bug back, so each call is pinned here by source text, +//! like the claim invariants in `session_commands.rs`. + +const LIB: &str = include_str!("../../lib.rs"); +const SETUP: &str = include_str!("../../workspace_setup.rs"); +const HOST: &str = include_str!("../host.rs"); +const CLONE: &str = include_str!("../clone.rs"); +const POLL: &str = include_str!("../poll.rs"); +const VERIFY: &str = include_str!("../verify.rs"); +const BRANCHES: &str = include_str!("../../branches.rs"); +const WEB_SERVER: &str = include_str!("../../web_server.rs"); +const SESSION_COMMANDS: &str = include_str!("../../session_commands.rs"); +const PR_CONTEXT: &str = include_str!("../../prs/context.rs"); +const PR_COMMIT: &str = include_str!("../../prs/commit.rs"); +const PR_GIT_PIPELINE: &str = include_str!("../../prs/git_pipeline.rs"); +const PR_PUSH: &str = include_str!("../../prs/push.rs"); +const PR_CREATE: &str = include_str!("../../prs/create_pr.rs"); +const PR_PULL: &str = include_str!("../../prs/pull.rs"); +const TRANSFER: &str = include_str!("../../project_transfer/orchestrator.rs"); + +/// The text of the function whose signature starts with `signature`, to the +/// closing brace at its own indentation. +fn body_of<'a>(source: &'a str, signature: &str) -> &'a str { + let at = source + .find(signature) + .unwrap_or_else(|| panic!("no `{signature}`")); + let line_start = source[..at].rfind('\n').map_or(0, |i| i + 1); + let indent: String = source[line_start..] + .chars() + .take_while(|c| *c == ' ') + .collect(); + let body = &source[at..]; + let end = body + .find(&format!("\n{indent}}}\n")) + .unwrap_or_else(|| panic!("no end of `{signature}`")); + &body[..end] +} + +/// The web server's dispatch arm for `command`, to the next arm. +fn web_arm(command: &str) -> &'static str { + let start = format!("\n \"{command}\" => {{"); + let at = WEB_SERVER + .find(&start) + .unwrap_or_else(|| panic!("no web arm for `{command}`")); + let arm = &WEB_SERVER[at + 1..]; + let end = arm.find("\n \"").expect("the next arm"); + &arm[..end] +} + +/// `first` is in `body`, and before `then`. +#[track_caller] +fn assert_before(what: &str, body: &str, first: &str, then: &str) { + let first_at = body + .find(first) + .unwrap_or_else(|| panic!("`{what}` no longer calls `{first}`")); + let then_at = body + .find(then) + .unwrap_or_else(|| panic!("`{what}` has no `{then}`")); + assert!( + first_at < then_at, + "`{what}` must call `{first}` before `{then}`" + ); +} + +#[track_caller] +fn assert_calls(what: &str, body: &str, call: &str) { + assert!(body.contains(call), "`{what}` no longer calls `{call}`"); +} + +// ── What triggers setup ───────────────────────────────────────────────────── + +/// The startup pass runs at launch, after interrupted transfers are +/// recovered, and does something for both kinds of row it lists: sets up +/// the ones left `Starting`, sets the retry of the ones left `Error`. +#[test] +fn the_startup_pass_is_run_and_acts_on_what_it_finds() { + assert_before( + "lib.rs", + LIB, + "project_transfer::recover_interrupted_transfers(", + "workspace_setup::resume_pending_setups(", + ); + + let pass = body_of(SETUP, "pub(crate) fn resume_pending_setups<"); + assert_calls( + "resume_pending_setups", + pass, + "branches_awaiting_setup(store)", + ); + assert_calls( + "resume_pending_setups", + pass, + "Some(StartupAction::SetUp) => trigger_branch_setup(app_handle, store, &branch.id)", + ); + assert_calls( + "resume_pending_setups", + pass, + "Some(StartupAction::ArmRetry) => arm_retry(app_handle, store, &branch.id)", + ); +} + +/// A trigger is an `Observed` setup of that branch, and a retry timer is +/// one on the shared guard. +#[test] +fn a_trigger_runs_setup_and_a_retry_is_armed_on_the_shared_guard() { + let trigger = body_of(SETUP, "pub(crate) fn trigger_branch_setup<"); + assert_calls( + "trigger_branch_setup", + trigger, + "ensure_branch_set_up(&app_handle, &store, &branch_id, SetupTrigger::Observed)", + ); + let ensure = body_of(SETUP, "pub(crate) async fn ensure_branch_set_up<"); + assert_calls("ensure_branch_set_up", ensure, "run_guarded("); + assert_calls("ensure_branch_set_up", ensure, "SetupGuard::global()"); + + let arm = body_of(SETUP, "pub(crate) fn arm_retry<"); + assert_calls("arm_retry", arm, "retry::arm("); + assert_calls("arm_retry", arm, "SetupGuard::global()"); + + let failed = body_of(SETUP, "pub(crate) fn branch_failed<"); + assert_calls( + "branch_failed", + failed, + "failures::set(app_handle, branch_id, Some(reason))", + ); + assert_calls( + "branch_failed", + failed, + "arm_retry(app_handle, store, branch_id)", + ); +} + +/// The real host announces failure reasons, runs the retry it is asked for, +/// and asks the reconcile service about a disconnected session. +#[test] +fn the_real_host_announces_retries_and_reconciles() { + let reason = body_of( + HOST, + " fn failure_changed(&self, branch_id: &str, error: Option<&str>) {\n failures", + ); + assert_calls( + "BloxHost::failure_changed", + reason, + "failures::set(&self.app_handle, branch_id, error)", + ); + + let later = body_of( + HOST, + " fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) {", + ); + assert_before( + "BloxHost::retry_later", + later, + "tokio::time::sleep(delay).await", + "retry::fire(SetupGuard::global(), &store, &host, &branch_id, ticket)", + ); + + let check = body_of( + HOST, + " fn check_disconnected_session(&self, session_id: &str) {", + ); + assert_calls( + "BloxHost::check_disconnected_session", + check, + "session_runner::reconcile::reconcile_session(", + ); +} + +/// Both pollers act through the app's effects, and those effects reach +/// setup: a branch seen `Starting` behind a running workstation is set up, +/// one seen in `Error` gets its retry, one the poll fails gets a reason. +#[test] +fn the_pollers_effects_reach_setup() { + let effects = body_of(POLL, "fn app_effects<"); + for call in [ + "branches::workspace_status_written(app_handle, branch_id, changed, status)", + "super::trigger_branch_setup(app_handle, store, branch_id)", + "super::arm_retry(app_handle, store, branch_id)", + "super::branch_failed(app_handle, store, branch_id, reason)", + ] { + assert_calls("app_effects", effects, call); + } + + let single = body_of(POLL, "pub(crate) async fn poll_workspace_status("); + assert_calls( + "poll_workspace_status", + single, + "&app_effects(store, app_handle)", + ); + let batch = body_of(POLL, "pub(crate) async fn poll_all_workspace_statuses("); + assert_calls( + "poll_all_workspace_statuses", + batch, + "&app_effects(store, app_handle)", + ); +} + +/// A resume that worked leaves its branches `Starting` and triggers setup +/// for each; one that failed gives each its reason and its retry. +#[test] +fn a_resume_triggers_setup_or_says_why_it_failed() { + let resume = body_of(BRANCHES, "pub(crate) async fn resume_workspace_impl("); + assert_calls( + "resume_workspace_impl", + resume, + "crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id)", + ); + assert_calls( + "resume_workspace_impl", + resume, + "crate::workspace_setup::branch_failed(app_handle, store, branch_id, &reason)", + ); + assert!( + !resume.contains("WorkspaceStatus::Running"), + "a resume must not say `Running`: setup does, after the checkout" + ); +} + +/// The command and its web arm ask for setup the same way and return the +/// row's resulting status rather than nothing. +#[test] +fn start_workspace_returns_what_setup_left_the_row_in() { + let command = body_of(BRANCHES, "pub async fn start_workspace("); + assert_calls( + "start_workspace", + command, + "Result", + ); + assert_calls( + "start_workspace", + command, + "crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id).await", + ); + assert_calls( + "the start_workspace web arm", + web_arm("start_workspace"), + "crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id)", + ); + + let request = body_of(SETUP, "pub(crate) async fn request_branch_setup<"); + assert_calls( + "request_branch_setup", + request, + "SetupRequest::new(SetupTrigger::Requested)", + ); + // Not held for a workstation's boot. + assert_calls("request_branch_setup", request, "Patience::WhileWorking"); + + assert_calls( + "the invoke handler", + LIB, + "workspace_setup::failures::get_workspace_setup_errors", + ); + assert_calls( + "the get_workspace_setup_errors web arm", + web_arm("get_workspace_setup_errors"), + "crate::workspace_setup::failures::get_workspace_setup_errors()", + ); +} + +// ── What writes `Running` ─────────────────────────────────────────────────── + +/// The text from each call of `write` in `source` to the end of its +/// statement. +fn status_writes<'a>(source: &'a str, write: &str) -> Vec<&'a str> { + source + .match_indices(write) + .map(|(at, _)| { + let call = &source[at..]; + &call[..call.find(';').unwrap_or(call.len())] + }) + .collect() +} + +/// Outside setup, nothing in the commands, the pollers, the session starts +/// or the web server writes a literal `Running`. Setup writes it in two +/// places: after a verified checkout, and when it puts back a `Running` +/// row whose repair was refused. +#[test] +fn only_setup_writes_running() { + for (name, source) in [ + ("branches.rs", BRANCHES), + ("workspace_setup/poll.rs", POLL), + ("workspace_setup/verify.rs", VERIFY), + ("session_commands.rs", SESSION_COMMANDS), + ("web_server.rs", WEB_SERVER), + ] { + let production = source.split("#[cfg(test)]").next().unwrap(); + for write in [ + "update_branch_workspace_status(", + "update_workspace_status_by_workspace_name(", + ] { + for call in status_writes(production, write) { + assert!( + !call.contains("WorkspaceStatus::Running"), + "{name} writes `Running` outside setup: {call}" + ); + } + } + } + + let running_writes = SETUP + .matches("update_branch_workspace_status(branch_id, &WorkspaceStatus::Running)") + .count(); + assert_eq!(running_writes, 2); + let set_up = body_of(SETUP, "pub(crate) async fn set_up_branch("); + assert_before( + "set_up_branch", + set_up, + "provision_and_check_out(", + "update_branch_workspace_status(branch_id, &WorkspaceStatus::Running)", + ); + let check_out = body_of(CLONE, "pub(super) async fn check_out_in_clone("); + assert_before( + "check_out_in_clone", + check_out, + "host.check_out(checkout).await?", + "verify::confirm_checked_out(host, checkout)", + ); +} + +/// A transfer that resumed a source workspace does not mark its rows +/// `Running` wholesale. It marks the ones its own inspection read as on +/// their branch, after that inspection. +#[test] +fn a_transfer_marks_running_only_the_rows_it_read_on_their_branch() { + assert!( + !TRANSFER.contains("update_workspace_status_by_workspace_name("), + "a transfer must not set a whole workspace's rows at once" + ); + let mark = body_of(TRANSFER, " fn mark_resumed_rows_running("); + assert_before( + "mark_resumed_rows_running", + mark, + "row.is_on_its_branch()", + "update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Running)", + ); + assert_eq!( + TRANSFER.matches("update_branch_workspace_status(").count(), + 1, + "a new status write in the transfer needs its own reading of the clone" + ); + assert_eq!( + TRANSFER.matches("self.mark_resumed_rows_running(").count(), + 1 + ); +} + +// ── What checks the clone before an agent or a pipeline starts ────────────── + +/// Every way a session starts on a branch reads the clone first, and keeps +/// the clone ([`super::super::verify::CheckoutHold`]) until its row is +/// live: the check comes before the claim or the launch, and its hold is +/// bound to a name, since `let _ =` would let go of it at once. +#[test] +fn every_session_start_checks_the_clone_before_it_launches() { + let starts = [ + ( + "pub(crate) async fn resume_session_for_store(", + "let _checkout = ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?;", + "launch_resume(", + ), + ( + "pub(crate) async fn resume_claimed_session_for_store(", + "Ok(plan) => ensure_resume_checkout(&app_handle, &store, &plan, &session.id)", + "launch_resume(", + ), + ( + "async fn start_requeued_session_for_branch(", + "let _checkout = ensure_resume_checkout(&app_handle, &store, &plan, &session_id).await?;", + "claim.claim(&store)", + ), + ( + "async fn start_queued_session_for_branch(", + "let _checkout = crate::workspace_setup::verify::ensure_session_checkout(", + "claim.claim(&store)", + ), + ]; + for (start, check, launch) in starts { + assert_before(start, body_of(SESSION_COMMANDS, start), check, launch); + } + // The claimed resume carries its hold out of the `match` by name. + let claimed = body_of( + SESSION_COMMANDS, + "pub(crate) async fn resume_claimed_session_for_store(", + ); + assert_before( + "resume_claimed_session_for_store", + claimed, + "Ok((plan, _checkout)) => plan,", + "launch_resume(", + ); + + // A direct start, whose session row does not exist yet. + let prepare = body_of(SESSION_COMMANDS, "async fn prepare_branch_session_start("); + assert_calls( + "prepare_branch_session_start", + prepare, + "crate::workspace_setup::verify::ensure_session_checkout(", + ); + + let resume_check = body_of(SESSION_COMMANDS, "async fn ensure_resume_checkout("); + assert_calls( + "ensure_resume_checkout", + resume_check, + "crate::workspace_setup::verify::ensure_session_checkout(app_handle, store, &branch, session_id)", + ); + let check = body_of(VERIFY, "pub(crate) async fn ensure_session_checkout<"); + assert_calls( + "ensure_session_checkout", + check, + "verify_session_checkout(SetupGuard::global(), store, &host, branch, session_id)", + ); +} + +/// The pipelines (commit, rebase, squash, push, pull request) run git in +/// the clone without an agent, and get the same check before their first +/// step: before the row is claimed or inserted as running. +#[test] +fn every_pipeline_checks_the_clone_before_its_git_steps() { + let starts = [ + ( + PR_COMMIT, + "pub(crate) async fn start_or_queue_commit_pipeline_for_branch(", + "insert_running_commit_pipeline_session(", + ), + ( + PR_COMMIT, + "pub(crate) async fn start_queued_commit_pipeline_for_branch(", + "claim.claim(&store)", + ), + ( + PR_GIT_PIPELINE, + "pub(crate) async fn start_queued_git_pipeline_for_branch(", + "claim.claim(&store)", + ), + ( + PR_PUSH, + "pub(crate) async fn start_or_queue_push_pipeline_for_branch(", + "insert_running_push_pipeline_session(", + ), + ( + PR_CREATE, + "pub(crate) async fn start_create_pr_pipeline_for_branch(", + "start_pipeline_for_branch(", + ), + ]; + for (source, start, launch) in starts { + let body = body_of(source, start); + assert_before( + start, + body, + "ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, ", + launch, + ); + assert!( + body.contains("let _checkout =") && !body.contains("let _ = ensure_pipeline_checkout"), + "`{start}` must keep the clone until its pipeline is in flight" + ); + } + + // The pull: checked in the task that runs it, before the pull itself. + let pull = body_of( + PR_PULL, + "pub(crate) async fn pull_or_queue_branch_for_branch(", + ); + assert_calls( + "pull_or_queue_branch_for_branch", + pull, + "ensure_pipeline_checkout(", + ); + let pull_with = body_of(PR_PULL, "pub(super) async fn pull_or_queue_branch_with("); + assert_before( + "pull_or_queue_branch_with", + pull_with, + "verify(session_id.clone()).await", + "spawn_blocking(pull)", + ); + + let check = body_of(PR_CONTEXT, "pub(crate) async fn ensure_pipeline_checkout("); + assert_calls( + "ensure_pipeline_checkout", + check, + "crate::workspace_setup::verify::ensure_session_checkout(", + ); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/verify.rs b/apps/staged/src-tauri/src/workspace_setup/verify.rs index 340c7da59..b37aad276 100644 --- a/apps/staged/src-tauri/src/workspace_setup/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/verify.rs @@ -7,12 +7,12 @@ //! //! - at the end of setup, before `Running` is written //! ([`confirm_checked_out`]); -//! - at every branch session start on a remote branch, a fresh session or -//! a resumed one, before the agent is launched -//! ([`ensure_session_checkout`]). A branch can be `Running` over a clone -//! setup never put there: a poller promotes a `Stopped` or `Suspended` row -//! when the workstation is resumed outside Staged, two branches of one -//! repo share a clone, someone switches branches in a terminal. +//! - at every start on a remote branch of something that runs git in the +//! clone, before it is launched ([`ensure_session_checkout`]): an agent +//! session, fresh or resumed, and a pipeline session (commit, push, +//! rebase, squash, PR). A branch can be `Running` over a clone that has +//! since moved: two branches of one repo share a clone, someone switches +//! branches in a terminal. //! //! The read is one transient exec and no fetch. It answers through stdout //! ([`CLONE_HEAD_PROBE_SCRIPT`]), so an exec that failed is not an answer: @@ -20,26 +20,35 @@ //! need) and never "right branch" (which would start the agent unchecked). //! [`read_head`] tries once more after a short pause and then fails. //! -//! A session start that finds the clone somewhere else repairs it rather -//! than failing: it runs the shared setup under the guard -//! ([`SetupTrigger::Repair`]) and starts once the clone reads right. Failing -//! instead would end every session queued on the branch one after another -//! with the same error and leave nothing that puts the clone right, since -//! nothing asks for setup of a `Running` branch. The repair moves the row to -//! `Starting` for as long as it runs and to `Error` when it fails, so the -//! session queue's own gate holds whatever else is queued until the branch -//! is usable. The one case that fails without a repair is a clone another -//! live session is working in: a checkout there would move that session's -//! work onto this branch. +//! A start that finds the clone somewhere else repairs it rather than +//! failing: it runs the shared setup under the guard +//! ([`SetupTrigger::Repair`]) and starts once the clone reads right. The +//! repair moves the row to `Starting` for as long as it runs, so the session +//! queue's own gate holds whatever else is queued. A repair that fails +//! leaves the row in `Error`, with its reason and a retry timer; one that is +//! refused, because switching would lose or move work or because another +//! live session is working in the clone, puts the row back to `Running` and +//! fails only the session that asked. +//! +//! The read and the start it allows are one step as far as other starts and +//! checkouts in the same clone go: the caller holds the clone's gate +//! ([`CheckoutHold`]) from the read until its session row is `running`. +//! Setup takes the same gate around its look for live sessions and its +//! checkout, so it either sees this session as live and leaves the clone +//! alone, or finishes before this session reads the clone. use std::sync::Arc; use std::time::Duration; +use tokio::sync::OwnedMutexGuard; + use crate::branches; use crate::store::{self, Store}; -use super::guard::SetupGuard; -use super::{checkout_for, run_guarded, BloxHost, Checkout, SetupHost, SetupTrigger}; +use super::clone::checkout_for; +use super::guard::{Patience, SetupGuard}; +use super::host::{blox_host, Checkout, SetupHost}; +use super::{run_guarded, SetupRequest}; /// What a clone has checked out. #[derive(Debug, Clone, PartialEq, Eq)] @@ -173,11 +182,11 @@ async fn read_head_once( } } -fn is_on_branch(head: &CloneHead, checkout: &Checkout) -> bool { +pub(super) fn is_on_branch(head: &CloneHead, checkout: &Checkout) -> bool { matches!(head, CloneHead::Branch(name) if *name == checkout.branch_name) } -fn unverified(checkout: &Checkout, error: &str) -> String { +pub(super) fn unverified(checkout: &Checkout, error: &str) -> String { format!( "Could not verify that branch '{}' is checked out in workspace '{}': {error}", checkout.branch_name, checkout.workspace @@ -201,74 +210,97 @@ pub(crate) async fn confirm_checked_out( } } -/// Make sure `branch`'s clone is on the branch before `session_id`'s agent -/// is launched in it. A no-op for a local branch. +/// The clone's gate, held by a start that has read the clone as on its +/// branch. While it is held no checkout runs in that clone and no other +/// start reads it. +/// +/// Keep it until the session's row is `running`, which is what setup looks +/// for before it touches a clone, and drop it then. Nothing may ask for +/// setup while holding one ([`SetupGuard::clone_gate`]'s lock order). +#[must_use = "the clone is only held while this is alive; bind it until the session row is running"] +pub(crate) struct CheckoutHold(#[allow(dead_code)] Option>); + +impl CheckoutHold { + /// For a start with no remote clone to hold: a local branch. + pub(crate) fn none() -> Self { + CheckoutHold(None) + } +} + +/// Make sure `branch`'s clone is on the branch before `session_id` starts +/// running git in it. A no-op for a local branch. /// /// Call it before anything else reads the clone for the session (the branch /// context in its prompt, its pre-run HEAD) and before the row is claimed: -/// it can wait on a repair. +/// it can wait on a repair. Bind what it returns until the row is `running` +/// ([`CheckoutHold`]). pub(crate) async fn ensure_session_checkout( app_handle: &tauri::AppHandle, store: &Arc, branch: &store::Branch, session_id: &str, -) -> Result<(), String> { +) -> Result { if branch.branch_type != store::BranchType::Remote || branch.workspace_name.is_none() { - return Ok(()); + return Ok(CheckoutHold::none()); } - let host: Arc = Arc::new(BloxHost { - app_handle: app_handle.clone(), - }); + let host = blox_host(app_handle, store); verify_session_checkout(SetupGuard::global(), store, &host, branch, session_id).await } /// [`ensure_session_checkout`] for a remote branch, against any host. /// -/// Returns `Ok` only after a read that says the clone is on the branch: -/// the first one, or the one after a repair. +/// Returns `Ok` only after a read, made with the clone's gate held, that +/// says the clone is on the branch: the first one, or the one after a +/// repair. pub(crate) async fn verify_session_checkout( guard: &Arc, store: &Arc, host: &Arc, branch: &store::Branch, session_id: &str, -) -> Result<(), String> { +) -> Result { let checkout = checkout_for(store, branch)?; - let found = match read_head(host.as_ref(), &checkout).await { - Ok(head) if is_on_branch(&head, &checkout) => return Ok(()), - Ok(head) => head, - Err(e) => return Err(unverified(&checkout, &e)), - }; + let clone_dir = checkout.clone_dir.as_deref(); let (workspace, expected) = (&checkout.workspace, &checkout.branch_name); + + let found = { + let gate = guard.clone_gate(workspace, clone_dir).await; + match read_head(host.as_ref(), &checkout).await { + Ok(head) if is_on_branch(&head, &checkout) => return Ok(CheckoutHold(Some(gate))), + Ok(head) => head, + Err(e) => return Err(unverified(&checkout, &e)), + } + // The gate is let go here: the repair takes it for its checkout. + }; log::warn!( "[workspace_setup] session {session_id}: workspace '{workspace}' has {found} checked \ - out, where branch '{expected}' was expected; checking the branch out before the agent \ - starts" + out, where branch '{expected}' was expected; checking the branch out before the \ + session starts" ); - if let Some(busy) = live_session_in_clone(store, branch, &checkout, session_id)? { - return Err(format!( - "Workspace '{workspace}' has {found} checked out, where this session's branch \ - '{expected}' was expected. It was not switched because a session of branch \ - '{busy}' is running in the same clone; start this session again once that one \ - has finished." - )); - } - - run_guarded(guard, store, host, &branch.id, SetupTrigger::Repair) - .await - .map_err(|e| { - format!( - "Workspace '{workspace}' had {found} checked out, where this session's branch \ - '{expected}' was expected, and checking the branch out failed: {e}" - ) - })?; + run_guarded( + guard, + store, + host, + &branch.id, + SetupRequest::repair_for(session_id), + Patience::ToTheEnd, + ) + .await + .map_err(|e| { + format!( + "Workspace '{workspace}' had {found} checked out, where this session's branch \ + '{expected}' was expected, and it could not be switched: {e}" + ) + })?; // The repair's own word is not taken for it: a run this one joined, or a // row that had left `Running` and so had nothing to repair, says nothing - // about where the clone is now. + // about where the clone is now. Neither does a repair that did its work: + // the gate was free between its checkout and this read. + let gate = guard.clone_gate(workspace, clone_dir).await; match read_head(host.as_ref(), &checkout).await { - Ok(head) if is_on_branch(&head, &checkout) => Ok(()), + Ok(head) if is_on_branch(&head, &checkout) => Ok(CheckoutHold(Some(gate))), Ok(head) => Err(format!( "Workspace '{workspace}' has {head} checked out, where this session's branch \ '{expected}' was expected" @@ -276,39 +308,3 @@ pub(crate) async fn verify_session_checkout( Err(e) => Err(unverified(&checkout, &e)), } } - -/// The branch whose live session works in `checkout`'s clone, if there is -/// one besides `session_id`: a session that is `running`, or `disconnected` -/// with its remote turn possibly still going. -/// -/// A project's remote branches share one workstation, and those of one -/// repository (the same repo added under two subpaths) share one clone -/// there. A checkout under a live agent would move the commits it goes on to -/// make onto another branch. -fn live_session_in_clone( - store: &Arc, - branch: &store::Branch, - checkout: &Checkout, - session_id: &str, -) -> Result, String> { - let branches_in_project = store - .list_branches_for_project(&branch.project_id) - .map_err(|e| e.to_string())?; - for other in branches_in_project { - if other.workspace_name.as_deref() != Some(checkout.workspace.as_str()) { - continue; - } - if other.id != branch.id - && branches::resolve_branch_clone_dir(store, &other)? != checkout.clone_dir - { - continue; - } - let live = store - .get_busy_sessions_for_branch(&other.id) - .map_err(|e| e.to_string())?; - if live.iter().any(|session| session.id != session_id) { - return Ok(Some(other.branch_name)); - } - } - Ok(None) -} diff --git a/apps/staged/src-tauri/src/workspace_setup/workstation.rs b/apps/staged/src-tauri/src/workspace_setup/workstation.rs new file mode 100644 index 000000000..686747812 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/workstation.rs @@ -0,0 +1,124 @@ +//! Bringing a workstation up for a branch: `sq blox ws start`, and the +//! bounded wait for one that is on its way. + +use std::sync::Arc; + +use crate::blox::BloxError; +use crate::branches; +use crate::store::{self, Store, WorkspaceStatus}; + +use super::guard::Progress; +use super::host::{reports_running, Checkout, SetupHost}; + +/// `sq blox ws start`, and when that fails without ruling the workstation +/// out, a bounded wait for it to come up anyway. +pub(super) async fn start_workstation( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, + checkout: &Checkout, + progress: &Progress, +) -> Result<(), String> { + let workspace = checkout.workspace.as_str(); + let source = format!( + "https://github.com/{}.git?ref={}", + checkout.repo_slug, checkout.base_ref + ); + log::info!( + "[workspace_setup] branch={} workspace={workspace} starting from {source}", + branch.id + ); + let start_error = match host.start_workspace(workspace, &source).await { + Ok(()) => return Ok(()), + Err(BloxError::NotAuthenticated) => { + return Err("Not authenticated with Blox. Run: sq login".to_string()); + } + // Definitive: keep the CLI's text, which carries the onboarding URL + // and the action it wants. + Err(e) if branches::is_blox_onboarding_precondition_error(&e) => { + return Err(e.to_string()); + } + Err(e) => e, + }; + + // `ws start` can fail — its own 120 s limit, a dropped connection — after + // Blox created the workstation, which then keeps booting. Returning here + // would leave the branch with a running workstation and no checkout, so + // wait for it and carry on, or fail where the user can see it. + log::warn!( + "[workspace_setup] branch={} workspace={workspace} ws start failed, waiting for the workstation: {start_error}", + branch.id + ); + wait_until_running( + store, + host, + branch, + workspace, + &format!("`sq blox ws start` failed: {start_error}"), + progress, + ) + .await +} + +/// Wait, within [`SetupHost::ready_wait`], for the workstation to report +/// running. `why` says what the wait was for, for the error when it never +/// does. +/// +/// The run says it is waiting for as long as this takes, so a caller that +/// only stays while the run is working ([`Patience::WhileWorking`]) leaves. +pub(super) async fn wait_until_running( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, + workspace: &str, + why: &str, + progress: &Progress, +) -> Result<(), String> { + progress.waiting_for_workstation(); + let result = poll_until_running(store, host, branch, workspace, why).await; + progress.working(); + result +} + +async fn poll_until_running( + store: &Arc, + host: &dyn SetupHost, + branch: &store::Branch, + workspace: &str, + why: &str, +) -> Result<(), String> { + let wait = host.ready_wait(); + let started = tokio::time::Instant::now(); + loop { + match host.workspace_status(workspace).await { + Ok(status) if reports_running(status.as_deref()) => return Ok(()), + Ok(status) + if branches::map_blox_status_to_workspace_status( + status.as_deref(), + Some(&WorkspaceStatus::Starting), + ) == WorkspaceStatus::Error => + { + return Err(format!( + "Workspace '{workspace}' failed to start (Blox reports it as {}); {why}", + status.as_deref().unwrap_or("error") + )); + } + Err(BloxError::NotAuthenticated) => { + return Err("Not authenticated with Blox. Run: sq login".to_string()); + } + // Still booting, or not listed yet. + Ok(_) | Err(_) => {} + } + // Deleted while it waited: nothing left to set up. + if store.get_branch(&branch.id).ok().flatten().is_none() { + return Err(format!("Branch not found: {}", branch.id)); + } + if started.elapsed() >= wait.timeout { + return Err(format!( + "Workspace '{workspace}' did not come up within {}s; {why}", + wait.timeout.as_secs() + )); + } + tokio::time::sleep(wait.interval).await; + } +} diff --git a/apps/staged/src/lib/commands.ts b/apps/staged/src/lib/commands.ts index 31f2e5368..7a759c92a 100644 --- a/apps/staged/src/lib/commands.ts +++ b/apps/staged/src/lib/commands.ts @@ -785,11 +785,27 @@ export function createRemoteBranch( }); } -/** Start the Blox workspace for a remote branch. */ -export function startWorkspace(branchId: string): Promise { +/** + * Set up a remote branch on its Blox workstation (or retry a branch in + * `error`). Resolves with the status the branch row holds afterwards: + * `running` once it is checked out and verified, `starting` when the backend + * returned early while the workstation boots (setup carries on there and its + * outcome lands on the row), or whatever else the row holds now. Rejects with + * the reason when setup failed and left the branch in `error`. + */ +export function startWorkspace(branchId: string): Promise { return invokeCommand('start_workspace', { branchId }); } +/** + * Why each remote branch's last setup run failed, by branch id, for a window + * that missed the `workspace-setup-error` events. Held in memory by the + * backend, so it is empty after an app restart. + */ +export function getWorkspaceSetupErrors(): Promise> { + return invokeCommand('get_workspace_setup_errors'); +} + /** Resume a suspended Blox workspace. Returns IDs of all affected branches. */ export function resumeWorkspace(workspaceName: string): Promise { return invokeCommand('resume_workspace', { workspaceName }); diff --git a/apps/staged/src/lib/features/branches/BranchCard.svelte b/apps/staged/src/lib/features/branches/BranchCard.svelte index e19841467..c11cb3a60 100644 --- a/apps/staged/src/lib/features/branches/BranchCard.svelte +++ b/apps/staged/src/lib/features/branches/BranchCard.svelte @@ -150,6 +150,7 @@ onRename?: (branchName: string) => void | Promise; onMove?: (targetProjectId: string) => void | Promise; onRetryWorktree?: () => void; + onRetryWorkspace?: () => void; } let { @@ -163,6 +164,7 @@ onRename, onMove, onRetryWorktree, + onRetryWorkspace, }: Props = $props(); // Determine if this is a local or remote branch @@ -2257,6 +2259,7 @@ status={remoteWorkspaceStatus} {workspaceError} fallbackError={error} + onRetry={onRetryWorkspace} /> {:else if loading && !isSettingUp}
diff --git a/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.component.test.ts b/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.component.test.ts new file mode 100644 index 000000000..455facae5 --- /dev/null +++ b/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.component.test.ts @@ -0,0 +1,49 @@ +import { flushSync, mount, unmount } from 'svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import RemoteWorkspaceStatusView from './RemoteWorkspaceStatusView.svelte'; +import type { WorkspaceStatus } from '../../types'; + +// A remote branch in `error` is only left by setting it up again, so the +// error view offers Retry whenever the card is given a way to do it. + +let component: ReturnType | null = null; +let target: HTMLElement; + +function render(props: { status: WorkspaceStatus; workspaceError?: string; onRetry?: () => void }) { + target = document.createElement('div'); + document.body.appendChild(target); + component = mount(RemoteWorkspaceStatusView, { target, props }); + flushSync(); +} + +function retryButton(): HTMLButtonElement | undefined { + return [...target.querySelectorAll('button')].find((b) => b.textContent?.trim() === 'Retry'); +} + +afterEach(() => { + if (component) unmount(component); + component = null; + target.remove(); +}); + +describe('RemoteWorkspaceStatusView', () => { + it('shows the reason and a Retry that calls onRetry in the error state', () => { + const onRetry = vi.fn(); + render({ status: 'error', workspaceError: 'could not fetch', onRetry }); + + expect(target.textContent).toContain('could not fetch'); + retryButton()?.click(); + flushSync(); + expect(onRetry).toHaveBeenCalledTimes(1); + }); + + it('has no Retry without onRetry, or outside the error state', () => { + render({ status: 'error', workspaceError: 'could not fetch' }); + expect(retryButton()).toBeUndefined(); + unmount(component!); + target.remove(); + + render({ status: 'suspended', onRetry: vi.fn() }); + expect(retryButton()).toBeUndefined(); + }); +}); diff --git a/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.svelte b/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.svelte index f3b49d4db..d17a07e8c 100644 --- a/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.svelte +++ b/apps/staged/src/lib/features/branches/RemoteWorkspaceStatusView.svelte @@ -1,6 +1,7 @@ {#if status === 'starting'} @@ -36,6 +39,9 @@ Workspace error {workspaceError ?? fallbackError ?? 'Something went wrong.'} + {#if onRetry} + + {/if}
{/if} diff --git a/apps/staged/src/lib/features/projects/ProjectHome.svelte b/apps/staged/src/lib/features/projects/ProjectHome.svelte index c54b675e4..3963d6dfe 100644 --- a/apps/staged/src/lib/features/projects/ProjectHome.svelte +++ b/apps/staged/src/lib/features/projects/ProjectHome.svelte @@ -996,6 +996,7 @@ onProjectTitleElement={selectedProjectId ? setProjectTitleElement : undefined} onRepoSelected={(selection) => handleRepoSelected(project.id, selection)} onRetryWorktree={(branchId) => setupBranchWorktree(branchId, project.id)} + onRetryWorkspace={(branchId) => workspaceLifecycle.retryWorkspace(project.id, branchId)} {tagPickerRequest} /> {/each} diff --git a/apps/staged/src/lib/features/projects/ProjectSection.svelte b/apps/staged/src/lib/features/projects/ProjectSection.svelte index c933cfb4d..d857974f3 100644 --- a/apps/staged/src/lib/features/projects/ProjectSection.svelte +++ b/apps/staged/src/lib/features/projects/ProjectSection.svelte @@ -93,6 +93,7 @@ onProjectTitleElement?: (element: HTMLHeadingElement | null) => void; onRepoSelected?: (selection: RepoPickerSelection) => void | Promise; onRetryWorktree?: (branchId: string) => void; + onRetryWorkspace?: (branchId: string) => void; /** Bumped to open the tag picker (the T shortcut, from ProjectHome). */ tagPickerRequest?: number; } @@ -113,6 +114,7 @@ onProjectTitleElement, onRepoSelected, onRetryWorktree, + onRetryWorkspace, tagPickerRequest = 0, }: Props = $props(); @@ -1236,6 +1238,7 @@ onRename={(branchName) => onRenameBranch?.(branch.id, branchName)} onMove={(targetProjectId) => onMoveBranch?.(branch.id, targetProjectId)} onRetryWorktree={() => onRetryWorktree?.(branch.id)} + onRetryWorkspace={onRetryWorkspace ? () => onRetryWorkspace(branch.id) : undefined} /> {/each} diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index 945445439..ff5283183 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -1,5 +1,6 @@ import type { Branch, ProjectLocation, WorkspaceStatus } from '../../types'; import * as commands from '../../api/commands'; +import { listenToEvent, type UnlistenFn } from '../../transport'; import { toast } from 'svelte-sonner'; import { getPreferredAgent } from '../settings/preferences.svelte'; import { agentState } from '../agents/agent.svelte'; @@ -17,6 +18,13 @@ interface RemoteSetupTask { run: () => Promise; } +/** Payload of the backend's `workspace-setup-error` event: why a setup run + * left the branch in `error`, or null when a new run for it begins. */ +interface WorkspaceSetupErrorEvent { + branchId: string; + error: string | null; +} + interface WorkspaceLifecycleHooks { getBranchesByProject: () => BranchMap; setBranchesByProject: (next: BranchMap) => void; @@ -40,6 +48,11 @@ class WorkspaceLifecycleController { private pendingSetupBranches = new Set(); private queuedSetupBranches = new Set(); + /** Remote branches whose `startWorkspace` answered `starting`: the backend + * returned early while their workstation boots and carries on setting + * them up itself, so nothing here re-asks, and the outcome arrives via + * the status poll (and the reason, on failure, via the error event). */ + private backendSetupBranches = new Set(); /** Projects a transfer just ended for, skipped until their refetch lands: * the lists held until then describe the old location. */ private awaitingTransferRefresh = new Set(); @@ -60,14 +73,27 @@ class WorkspaceLifecycleController { // any is running, stopped otherwise. private readonly STARTING_POLL_MS = 3000; private readonly RUNNING_POLL_MS = 30_000; - // How long a branch may sit in `starting` with nothing here setting it up + // How long a branch may sit in `starting` with nothing setting it up // before it is shown as failed. A branch stays `starting` until the // backend has checked it out on the workstation, not just until the - // workstation is up, and that setup is bounded and reports its own - // failure through `startWorkspace`. So the clock only runs for a branch - // this controller has no setup call queued or in flight for. + // workstation is up. That setup is bounded and lands its own outcome on + // the row (`running`, or `error` with a reason sent as a + // `workspace-setup-error` event, and as a rejection of a `startWorkspace` + // call joined to it). So the clock only runs for a branch with no setup + // call queued or in flight here, and whose last `startWorkspace` did not + // answer `starting` (the backend returning early while the workstation + // boots, and carrying on with the setup itself). private readonly STARTING_TIMEOUT_MS = 5 * 60 * 1000; + /** Bumped by start() and stop(), so an answer arriving after a restart + * does not write into the next run's state. */ + private generation = 0; + private unlistenSetupErrors: UnlistenFn | null = null; + /** Counts setup-error events, and the count at each branch's latest one, + * so a setup-error snapshot does not overwrite a newer event. */ + private setupErrorEventSeq = 0; + private setupErrorEventSeqByBranch = new Map(); + private pollTimer: ReturnType | null = null; private pollInFlight = false; private startingTimestamps = new Map(); @@ -75,10 +101,28 @@ class WorkspaceLifecycleController { start(hooks: WorkspaceLifecycleHooks): void { this.hooks = hooks; + const generation = ++this.generation; + this.unlistenSetupErrors?.(); + // The backend reports every failed setup run, including the ones no call + // from here was joined to (its startup pass, poll-triggered runs and + // automatic retries). The snapshot covers failures from before this + // listener went live, and is retaken on each web-socket reconnect. + this.unlistenSetupErrors = listenToEvent( + 'workspace-setup-error', + (payload) => { + if (this.generation !== generation) return; + this.applySetupErrorEvent(payload); + }, + { onEstablished: () => void this.seedSetupErrors(generation) } + ); this.ensurePolling(); } stop(): void { + this.generation++; + this.unlistenSetupErrors?.(); + this.unlistenSetupErrors = null; + this.setupErrorEventSeqByBranch.clear(); this.stopPolling(); if (this.kickoffTimer) { clearTimeout(this.kickoffTimer); @@ -86,6 +130,7 @@ class WorkspaceLifecycleController { } this.pendingSetupBranches.clear(); this.queuedSetupBranches.clear(); + this.backendSetupBranches.clear(); this.awaitingTransferRefresh.clear(); this.prerunOnSetup.clear(); this.activeRemoteWorkspaceStarts.clear(); @@ -150,8 +195,10 @@ class WorkspaceLifecycleController { if (!branches) return; let changed = false; + let wasError = false; const nextBranches = branches.map((branch) => { if (branch.id !== branchId) return branch; + wasError = branch.workspaceStatus === 'error'; const statusChanged = branch.workspaceStatus !== workspaceStatus; const idChanged = workstationId != null && branch.workstationId !== workstationId; if (!statusChanged && !idChanged) return branch; @@ -167,11 +214,15 @@ class WorkspaceLifecycleController { hooks.setBranchesByProject(new Map(current).set(projectId, nextBranches)); } - if (workspaceStatus !== 'error' && this.workspaceErrors.has(branchId)) { - const nextWorkspaceErrors = new Map(this.workspaceErrors); - nextWorkspaceErrors.delete(branchId); - this.workspaceErrors = nextWorkspaceErrors; - this.version++; + // The reason goes once the branch is out of `error` or set up. Not on any + // non-error status: a reason the error event delivered can precede the + // poll that reports the branch's `error`, behind a stale `starting`. + if (workspaceStatus === 'running' || (wasError && workspaceStatus !== 'error')) { + this.clearWorkspaceError(branchId); + } + + if (workspaceStatus !== 'starting') { + this.backendSetupBranches.delete(branchId); } // Track starting timestamps for timeout logic. @@ -195,6 +246,31 @@ class WorkspaceLifecycleController { await this.setupBranchWorktree(branchId, projectId, { runPrerun: true }); } + /** + * Set a remote branch up again, typically one in `error`: the backend puts + * it back to `starting` and reruns setup. A no-op while a setup call for + * the branch is already queued or in flight here. + */ + retryWorkspace(projectId: string, branchId: string): void { + const hooks = this.hooks; + if (!hooks || this.isProjectHeld(projectId)) return; + if (this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId)) return; + const branch = hooks + .getBranchesByProject() + .get(projectId) + ?.find((b) => b.id === branchId); + if (!branch || branch.branchType !== 'remote') return; + // As in enqueueBranchSetup: a list read before a transfer switched the + // project over would set up the old location again. + const location = hooks.getProjectLocation(projectId); + if (location && location !== 'remote') return; + + this.backendSetupBranches.delete(branchId); + this.clearWorkspaceError(branchId); + this.handleWorkspaceStatusChange(projectId, branchId, 'starting'); + this.queueRemoteSetup(projectId, branchId, branch.workspaceName); + } + async resumeWorkspace(projectId: string, workspaceName: string): Promise { // Optimistically transition all peer branches sharing this workspace. const hooks = this.hooks; @@ -259,6 +335,7 @@ class WorkspaceLifecycleController { clearBranchState(branchId: string): void { this.pendingSetupBranches.delete(branchId); this.queuedSetupBranches.delete(branchId); + this.backendSetupBranches.delete(branchId); this.startingTimestamps.delete(branchId); this.prerunOnSetup.delete(branchId); @@ -296,9 +373,56 @@ class WorkspaceLifecycleController { ); } - /** Whether a setup call for the branch is queued or in flight here. */ + /** Whether a setup call for the branch is queued or in flight here, or + * the backend said it carries on with the branch's setup itself. */ private isSetupOwned(branchId: string): boolean { - return this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId); + return ( + this.pendingSetupBranches.has(branchId) || + this.queuedSetupBranches.has(branchId) || + this.backendSetupBranches.has(branchId) + ); + } + + private setWorkspaceError(branchId: string, message: string): void { + if (this.workspaceErrors.get(branchId) === message) return; + this.workspaceErrors = new Map(this.workspaceErrors).set(branchId, message); + this.version++; + } + + private clearWorkspaceError(branchId: string): void { + if (!this.workspaceErrors.has(branchId)) return; + const next = new Map(this.workspaceErrors); + next.delete(branchId); + this.workspaceErrors = next; + this.version++; + } + + /** Not toasted: a failure a call from here was joined to is toasted when + * that call rejects. */ + private applySetupErrorEvent({ branchId, error }: WorkspaceSetupErrorEvent): void { + this.setupErrorEventSeqByBranch.set(branchId, ++this.setupErrorEventSeq); + if (error == null) { + this.clearWorkspaceError(branchId); + } else { + this.setWorkspaceError(branchId, error); + } + } + + private async seedSetupErrors(generation: number): Promise { + const seqAtRequest = this.setupErrorEventSeq; + let errors: Record; + try { + errors = await commands.getWorkspaceSetupErrors(); + } catch (e) { + console.error('[workspaceLifecycle] Failed to load workspace setup errors:', e); + return; + } + if (this.generation !== generation) return; + for (const [branchId, message] of Object.entries(errors ?? {})) { + // An event since the request is newer than the snapshot. + if ((this.setupErrorEventSeqByBranch.get(branchId) ?? 0) > seqAtRequest) continue; + this.setWorkspaceError(branchId, message); + } } private errorMessage(err: unknown): string { @@ -446,7 +570,12 @@ class WorkspaceLifecycleController { private enqueueBranchSetup(projectId: string, branch: Branch): void { const branchId = branch.id; - if (this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId)) return; + // The backend's own setup run is over once the row leaves `starting`, + // whichever of the poll or a refetched list reports it first. + if (branch.branchType === 'remote' && branch.workspaceStatus !== 'starting') { + this.backendSetupBranches.delete(branchId); + } + if (this.isSetupOwned(branchId)) return; // A project's branches all live where it does. One that doesn't is from // a list read before a transfer switched the project over, and setting @@ -469,42 +598,66 @@ class WorkspaceLifecycleController { } if (branch.branchType === 'remote' && branch.workspaceStatus === 'starting') { - this.queuedSetupBranches.add(branchId); - this.remoteSetupTaskQueue.push({ - branchId, - workspaceName: branch.workspaceName, - run: async () => { - this.pendingSetupBranches.add(branchId); - if (this.workspaceErrors.has(branchId)) { - const nextWorkspaceErrors = new Map(this.workspaceErrors); - nextWorkspaceErrors.delete(branchId); - this.workspaceErrors = nextWorkspaceErrors; - this.version++; - } + this.queueRemoteSetup(projectId, branchId, branch.workspaceName); + } + } - try { - // Resolves once the backend has the branch checked out on the - // workstation and marked `running`; rejects with why it is in - // `error`. A poll never promotes a `starting` branch, so this - // answer is what moves the branch on. - await commands.startWorkspace(branchId); - this.handleWorkspaceStatusChange(projectId, branchId, 'running'); - } catch (e) { - console.error('[workspaceLifecycle] Failed to start workspace:', e); - const message = this.errorMessage(e); - this.workspaceErrors = new Map(this.workspaceErrors).set(branchId, message); - this.version++; - this.handleWorkspaceStatusChange(projectId, branchId, 'error'); - toast.error('Unable to start workspace', { - description: message, - duration: Infinity, - }); - } finally { - this.pendingSetupBranches.delete(branchId); - } - }, + private queueRemoteSetup( + projectId: string, + branchId: string, + workspaceName: string | null + ): void { + this.queuedSetupBranches.add(branchId); + this.remoteSetupTaskQueue.push({ + branchId, + workspaceName, + run: () => this.runRemoteSetup(projectId, branchId), + }); + // A branch can turn `starting` in a refetched list (the backend retrying + // it on its own) with the poller stopped; its outcome comes from polls. + this.ensurePolling(); + this.pumpRemoteSetupQueue(); + } + + private async runRemoteSetup(projectId: string, branchId: string): Promise { + const generation = this.generation; + this.pendingSetupBranches.add(branchId); + this.clearWorkspaceError(branchId); + + try { + // Joins the backend's setup run for the branch (starting one if none + // is running). Resolves with the row's status: `running` once the + // branch is checked out and verified, `starting` when the backend + // returned early while the workstation boots, or whatever else the + // row holds now. Rejects with why it is in `error`. + const result = await commands.startWorkspace(branchId); + if (this.generation !== generation) return; + const status = this.toWorkspaceStatus(result?.status ?? ''); + if (!status) { + // Nothing to apply: leave the branch to the poll, without asking + // again on every kickoff meanwhile. + console.warn('[workspaceLifecycle] Unknown startWorkspace status:', result); + this.backendSetupBranches.add(branchId); + this.ensurePolling(); + return; + } + if (status === 'starting') { + // Setup carries on in the backend: the poll reports how it ends. + this.backendSetupBranches.add(branchId); + } + this.handleWorkspaceStatusChange(projectId, branchId, status, result.workstationId); + } catch (e) { + if (this.generation !== generation) return; + console.error('[workspaceLifecycle] Failed to start workspace:', e); + const message = this.errorMessage(e); + this.setWorkspaceError(branchId, message); + this.handleWorkspaceStatusChange(projectId, branchId, 'error'); + toast.error('Unable to start workspace', { + description: message, + duration: Infinity, }); - this.pumpRemoteSetupQueue(); + } finally { + if (this.generation === generation) this.pendingSetupBranches.delete(branchId); } } diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 0ef428de4..89a004a2d 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -10,6 +10,15 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { let transferringProjects: Set; let projectLocations: Map; let refreshTransferredProject: Mock<(projectId: string) => Promise>; + let eventListeners: Map void>>; + + /** Deliver a backend event to the controller's listeners. */ + function emit(event: string, payload: unknown) { + for (const listener of eventListeners.get(event) ?? []) listener(payload); + } + + const RUNNING = { status: 'running', workstationId: null }; + const STARTING = { status: 'starting', workstationId: null }; beforeEach(() => { vi.resetModules(); @@ -25,10 +34,30 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { commandsMock = { setupWorktree: vi.fn(), setupWorktreeAndRunPrerun, - startWorkspace: vi.fn(async () => {}), + startWorkspace: vi.fn(async () => RUNNING), pollAllWorkspaceStatuses: vi.fn(async () => ({})), + getWorkspaceSetupErrors: vi.fn(async () => ({})), }; vi.doMock('../../api/commands', () => commandsMock); + eventListeners = new Map(); + vi.doMock('../../transport', () => ({ + listenToEvent: ( + event: string, + callback: (payload: unknown) => void, + opts?: { onEstablished?: () => void } + ) => { + const listeners = eventListeners.get(event) ?? []; + listeners.push(callback); + eventListeners.set(event, listeners); + void Promise.resolve().then(() => opts?.onEstablished?.()); + return () => { + eventListeners.set( + event, + (eventListeners.get(event) ?? []).filter((l) => l !== callback) + ); + }; + }, + })); vi.doMock('svelte-sonner', () => ({ toast: { error: vi.fn() } })); vi.doMock('../settings/preferences.svelte', () => ({ getPreferredAgent: () => 'claude', @@ -45,6 +74,7 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { vi.useRealTimers(); vi.unstubAllGlobals(); vi.doUnmock('../../api/commands'); + vi.doUnmock('../../transport'); vi.doUnmock('svelte-sonner'); vi.doUnmock('../settings/preferences.svelte'); vi.doUnmock('../agents/agent.svelte'); @@ -257,9 +287,9 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { // it would show an error over a setup that then succeeds. it('does not time out a starting branch while its setup call is in flight', async () => { vi.useFakeTimers(); - let finishSetup!: () => void; + let finishSetup!: (result: typeof RUNNING) => void; commandsMock.startWorkspace.mockReturnValue( - new Promise((resolve) => (finishSetup = resolve)) + new Promise((resolve) => (finishSetup = resolve)) ); // The poll never promotes it: only setup does. commandsMock.pollAllWorkspaceStatuses.mockResolvedValue({ @@ -276,7 +306,7 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('starting'); expect(lifecycle.getWorkspaceErrors().size).toBe(0); - finishSetup(); + finishSetup(RUNNING); await settle(); expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('running'); lifecycle.stop(); @@ -341,10 +371,147 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { expect(branchesByProject.get('project-1')).toEqual([ { id: 'branch-1', branchType: 'remote', workspaceStatus: 'running' }, ]); + // The setup-error snapshot taken at start is a read, not a drain. const issued = Object.entries(commandsMock) - .filter(([, fn]) => fn.mock.calls.length > 0) + .filter(([name, fn]) => fn.mock.calls.length > 0 && name !== 'getWorkspaceSetupErrors') .map(([name]) => name); expect(issued).toEqual(['pollAllWorkspaceStatuses']); lifecycle.stop(); }); + /** Remote branches in `starting`, as the store would hold them. */ + function startingBranches(...ids: string[]) { + return ids.map((id) => ({ id, branchType: 'remote', workspaceStatus: 'starting' })); + } + + function statusOf(branchId: string) { + return branchesByProject.get('project-1')?.find((b) => b.id === branchId)?.workspaceStatus; + } + + // The resolved status is the row's, not an inference: a setup call can + // come back with the branch suspended (or anything else) as well. + it('applies the status startWorkspace resolves with', async () => { + vi.useFakeTimers(); + commandsMock.startWorkspace.mockImplementation(async (branchId: string) => + branchId === 'branch-1' + ? { status: 'running', workstationId: 7 } + : { status: 'suspended', workstationId: null } + ); + branchesByProject = new Map([['project-1', startingBranches('branch-1', 'branch-2')]]); + const lifecycle = await newController(); + await kickoff(lifecycle); + + expect(branchesByProject.get('project-1')).toEqual([ + { id: 'branch-1', branchType: 'remote', workspaceStatus: 'running', workstationId: 7 }, + { id: 'branch-2', branchType: 'remote', workspaceStatus: 'suspended' }, + ]); + lifecycle.stop(); + }); + + // `starting` means the backend returned early while the workstation boots + // and carries on with the setup itself. The call must not keep a setup + // slot, be re-issued on the next kickoff, or let the timeout fire under + // the backend's run; the poll reports how it ends. + it('releases the slot on a starting answer and leaves the outcome to the poll', async () => { + vi.useFakeTimers(); + const ids = ['branch-1', 'branch-2', 'branch-3', 'branch-4', 'branch-5']; + commandsMock.startWorkspace.mockImplementation((branchId: string) => + branchId === 'branch-5' ? new Promise(() => {}) : Promise.resolve(STARTING) + ); + let polled: Record = + Object.fromEntries(ids.map((id) => [id, STARTING])); + commandsMock.pollAllWorkspaceStatuses.mockImplementation(async () => polled); + branchesByProject = new Map([['project-1', startingBranches(...ids)]]); + const lifecycle = await newController(); + await kickoff(lifecycle); + + expect(commandsMock.startWorkspace.mock.calls.map(([id]) => id)).toEqual(ids); + await kickoff(lifecycle); + expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(5); + + await vi.advanceTimersByTimeAsync(7 * 60 * 1000); + expect(ids.map(statusOf)).toEqual(ids.map(() => 'starting')); + expect(lifecycle.getWorkspaceErrors().size).toBe(0); + + polled = { ...polled, 'branch-1': { status: 'running', workstationId: 3 } }; + await vi.advanceTimersByTimeAsync(3000); + expect(statusOf('branch-1')).toBe('running'); + lifecycle.stop(); + }); + + // A list refetched after the backend retried a branch on its own (error to + // starting) joins that run and polls it, even with the poller stopped. + it('joins and polls a branch a refetched list shows starting again', async () => { + vi.useFakeTimers(); + commandsMock.startWorkspace.mockResolvedValue(STARTING); + branchesByProject = new Map([ + ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'error' }]], + ]); + const lifecycle = await newController(); + await kickoff(lifecycle); + expect(commandsMock.startWorkspace).not.toHaveBeenCalled(); + expect(commandsMock.pollAllWorkspaceStatuses).not.toHaveBeenCalled(); + + branchesByProject = new Map([['project-1', startingBranches('branch-1')]]); + await kickoff(lifecycle); + expect(commandsMock.startWorkspace).toHaveBeenCalledWith('branch-1'); + expect(commandsMock.pollAllWorkspaceStatuses).toHaveBeenCalledWith(['branch-1']); + lifecycle.stop(); + }); + + it('retryWorkspace reruns setup for a branch in error, once at a time', async () => { + vi.useFakeTimers(); + commandsMock.startWorkspace.mockRejectedValueOnce(new Error('could not fetch')); + branchesByProject = new Map([['project-1', startingBranches('branch-1')]]); + const lifecycle = await newController(); + await kickoff(lifecycle); + expect(statusOf('branch-1')).toBe('error'); + expect(lifecycle.getWorkspaceErrors().get('branch-1')).toBe('could not fetch'); + + let finishSetup!: (result: typeof RUNNING) => void; + commandsMock.startWorkspace.mockReturnValueOnce( + new Promise((resolve) => (finishSetup = resolve)) + ); + lifecycle.retryWorkspace('project-1', 'branch-1'); + await settle(); + expect(statusOf('branch-1')).toBe('starting'); + expect(lifecycle.getWorkspaceErrors().has('branch-1')).toBe(false); + expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(2); + + lifecycle.retryWorkspace('project-1', 'branch-1'); + await settle(); + expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(2); + + finishSetup(RUNNING); + await settle(); + expect(statusOf('branch-1')).toBe('running'); + lifecycle.stop(); + }); + + // The backend reports failures of runs nothing here joined (its startup + // pass, poll triggers, automatic retries), and a snapshot covers the ones + // from before the window mounted. + it('keeps setup reasons from the error event and the snapshot taken at start', async () => { + let finishSnapshot!: (errors: Record) => void; + commandsMock.getWorkspaceSetupErrors.mockReturnValue( + new Promise((resolve) => (finishSnapshot = resolve)) + ); + const lifecycle = await newController(); + await settle(); + expect(commandsMock.getWorkspaceSetupErrors).toHaveBeenCalledTimes(1); + + // An event since the snapshot was asked for is newer than it. + emit('workspace-setup-error', { branchId: 'branch-2', error: null }); + finishSnapshot({ 'branch-1': 'workstation never booted', 'branch-2': 'stale' }); + await settle(); + expect([...lifecycle.getWorkspaceErrors()]).toEqual([['branch-1', 'workstation never booted']]); + + emit('workspace-setup-error', { branchId: 'branch-3', error: 'could not fetch' }); + expect(lifecycle.getWorkspaceErrors().get('branch-3')).toBe('could not fetch'); + emit('workspace-setup-error', { branchId: 'branch-3', error: null }); + expect(lifecycle.getWorkspaceErrors().has('branch-3')).toBe(false); + + // Stopped: the listener is gone. + lifecycle.stop(); + expect(eventListeners.get('workspace-setup-error')).toEqual([]); + }); }); From 49fe5605eadcb76fee725e63490dc6fbb3199dd5 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Wed, 18 Nov 2026 14:06:10 +1100 Subject: [PATCH 4/9] refactor(staged): let the backend own branch setup and keep automatic retries quiet The second review of this branch asked for less machinery around setup and for retries that nobody asked for to stop being visible. This applies its simplification of workspace_setup.rs, its first warning and the two small suggestions on the frontend timeout and on stale failure reasons. The other four warnings are left for a later commit. start_workspace now only asks. It returns once the run is under way or joined, the row and the workspace-setup-error event say how the run ends, and no client call waits on a setup. That removes the Patience and Progress machinery, the Queued and WaitingForWorkstation guard states and the early-return branch in the backend, and backendSetupBranches, the four-slot remote queue and the five-minute starting timeout in the frontend. The invariant is unchanged: Running is written only by setup, over a clone it read. Failures reach the UI through the reason event and query. The event carries userRequested, set for a setup the user asked for (a new branch, the error view's Retry, a resume), and the frontend raises a toast only for those. A startup pass, a poll-triggered run and an automatic retry fail into the card only. An automatic retry no longer flips the card or blanks its reason: the row stays Error with its reason until the checkout has passed. The timer looks at the workstation before taking a slot in the guard, which shrinks the joined-run loop to one second look. A retry that the user's Retry or a resume joined while it waited for its turn becomes that run. A checkout refused over what is in the clone sets no timer and is left to the Retry button; a live session or a workstation that is down still does (CheckoutError::Blocked). Retry timers wait up to a fifth longer, at random, so a restart does not retry every failed branch in one instant. A stored reason is dropped when its branch is gone or no longer in Error, at the retry timer and whenever a failure is recorded, and a reason that has not changed is not announced again. Tests: the request, retry and wiring tests are adapted to the new entry points, with new ones for a request that returns at once, a quiet failed retry, a refusal that sets no timer, jitter, what the reason store announces, and a frontend told about an automatic retry failing without a toast. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- apps/staged/src-tauri/src/branches.rs | 19 +- apps/staged/src-tauri/src/web_server.rs | 12 +- apps/staged/src-tauri/src/workspace_setup.rs | 387 ++++++++---------- .../src-tauri/src/workspace_setup/checkout.rs | 19 +- .../src-tauri/src/workspace_setup/clone.rs | 2 +- .../src-tauri/src/workspace_setup/failures.rs | 72 +++- .../src-tauri/src/workspace_setup/guard.rs | 219 ++++------ .../src-tauri/src/workspace_setup/host.rs | 11 +- .../src-tauri/src/workspace_setup/poll.rs | 2 +- .../src-tauri/src/workspace_setup/retry.rs | 70 +++- .../src-tauri/src/workspace_setup/tests.rs | 68 +-- .../src/workspace_setup/tests/checkout.rs | 3 +- .../src/workspace_setup/tests/failures.rs | 59 +++ .../src/workspace_setup/tests/requests.rs | 165 ++++---- .../src/workspace_setup/tests/retry.rs | 90 +++- .../src/workspace_setup/tests/wiring.rs | 56 +-- .../src-tauri/src/workspace_setup/verify.rs | 24 +- .../src/workspace_setup/workstation.rs | 20 - apps/staged/src/lib/commands.ts | 15 +- .../projects/workspaceLifecycle.svelte.ts | 243 +++-------- .../projects/workspaceLifecycle.test.ts | 194 ++++----- 21 files changed, 860 insertions(+), 890 deletions(-) create mode 100644 apps/staged/src-tauri/src/workspace_setup/tests/failures.rs diff --git a/apps/staged/src-tauri/src/branches.rs b/apps/staged/src-tauri/src/branches.rs index 2354da904..b0af7ca85 100644 --- a/apps/staged/src-tauri/src/branches.rs +++ b/apps/staged/src-tauri/src/branches.rs @@ -2378,21 +2378,19 @@ pub(crate) async fn create_remote_branch_impl( /// /// Also the error view's Retry: a branch in `Error` is set up again. /// -/// Returns the status the branch's row holds, so the caller does not infer -/// one from the call having resolved: `running` once the branch is checked -/// out on the workstation, or `starting` when setup is still waiting for the -/// workstation to boot, which it goes on doing after this returns. An error -/// is the failure that left the branch in `Error`. The shared routine in -/// [`crate::workspace_setup`], which a concurrent call for the same branch -/// joins. +/// Only asks: this returns once the setup run is under way, joined to the +/// one in flight when there is one. The backend carries the run through, its +/// outcome lands on the branch's row, and a failure's reason is announced as +/// a `workspace-setup-error` event. An error here means the request could +/// not be made. See [`crate::workspace_setup::request_setup`]. #[tauri::command(rename_all = "camelCase")] pub async fn start_workspace( store: tauri::State<'_, Mutex>>>, app_handle: AppHandle, branch_id: String, -) -> Result { +) -> Result<(), String> { let store = get_store(&store)?; - crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id).await + crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id) } /// Return the value of the `BLOX_ENV` environment variable, if set. @@ -2625,8 +2623,9 @@ pub(crate) async fn resume_workspace_impl( return Err(e.to_string()); } + // The user's resume: a setup that fails after it is theirs to hear of. for branch_id in &branch_ids { - crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id); + crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id, true); } Ok(branch_ids) diff --git a/apps/staged/src-tauri/src/web_server.rs b/apps/staged/src-tauri/src/web_server.rs index d3767ae74..162c45d94 100644 --- a/apps/staged/src-tauri/src/web_server.rs +++ b/apps/staged/src-tauri/src/web_server.rs @@ -2244,14 +2244,9 @@ pub(crate) async fn dispatch( "start_workspace" => { let store = get_store(store_mutex)?; let branch_id: String = arg(&args, "branchId")?; - let app_handle = app_handle.clone(); - detached(async move { - let result = - crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id) - .await?; - Ok(serde_json::to_value(result).unwrap()) - }) - .await + // Only asks: the run it starts or joins is its own task. + crate::workspace_setup::request_branch_setup(app_handle, &store, &branch_id)?; + Ok(Value::Null) } "get_workspace_setup_errors" => { let errors = crate::workspace_setup::failures::get_workspace_setup_errors(); @@ -5092,7 +5087,6 @@ mod tests { ]; /// Arms whose bodies run through `detached`. const DETACHED: &[&str] = &[ - "start_workspace", "resume_workspace", "pull_or_queue_branch", "resume_session", diff --git a/apps/staged/src-tauri/src/workspace_setup.rs b/apps/staged/src-tauri/src/workspace_setup.rs index 97d2bc29e..47565233b 100644 --- a/apps/staged/src-tauri/src/workspace_setup.rs +++ b/apps/staged/src-tauri/src/workspace_setup.rs @@ -9,14 +9,19 @@ //! `Running` on a workstation still on its base branch has its sessions //! commit onto the base. //! +//! The backend owns setup end to end. A client only asks for it and renders +//! what the row and the reason event say; nothing waits on a client. +//! //! A failure leaves the branch in `Error`, and `Error` always has a way out: //! the reason is recorded and announced ([`failures`]), a retry timer is set -//! ([`retry`]), and the error view's Retry asks for setup again. +//! unless the checkout refused over what is in the clone ([`retry`]), and +//! the error view's Retry asks for setup again. //! //! Setup is level-triggered. Whoever sees a branch that needs it asks: //! //! - the frontend, through the `start_workspace` command, for every -//! `Starting` branch it holds and for an `Error` one the user retries; +//! `Starting` branch it holds and for an `Error` one the user retries +//! ([`request_setup`]). The call returns once the run is under way; //! - the status pollers ([`poll`]), which see a running workstation behind a //! row that doesn't say `Running`. They never promote it: a `Starting` row //! gets setup triggered, a `Stopped`, `Suspended` or statusless one is @@ -24,7 +29,8 @@ //! - `resume_workspace`, once the workstation is back; //! - the startup pass ([`resume_pending_setups`]), for branches a previous //! run left in `Starting` or `Error`; -//! - a retry timer ([`SetupTrigger::Retry`]). +//! - a retry timer ([`SetupTrigger::Retry`]), which is quiet: the row stays +//! `Error` with its reason until the checkout has passed. //! //! One more caller asks for a branch that is already `Running`: an agent or //! pipeline start that read the clone and found it off the branch @@ -32,10 +38,8 @@ //! //! All of them go through one [`guard::SetupGuard`], so a second request for //! a branch joins the run in flight, and two branches never run setup in one -//! workstation at once. A caller that joined a run started for another -//! reason does not take that run's "nothing to do" for its own answer -//! ([`run_guarded`]). The guard is per process: a second Staged instance on -//! the same database runs its own. +//! workstation at once. The guard is per process: a second Staged instance +//! on the same database runs its own. //! //! The checkout itself ([`checkout`]) never loses work or moves it between //! branches, and never runs in a clone a live session is working in @@ -55,13 +59,11 @@ mod workstation; use std::sync::Arc; -use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; -use crate::PollWorkspaceResult; use checkout::CheckoutError; use clone::{check_out_in_clone, checkout_for}; -use guard::{Patience, Progress, SetupGuard}; +use guard::SetupGuard; use host::{blox_host, reports_booting, reports_running, SetupHost}; use verify::CloneHead; use workstation::{start_workstation, wait_until_running}; @@ -77,9 +79,10 @@ pub(crate) enum SetupTrigger { /// the startup pass. It acts only while the row still says so, so a /// trigger that arrives after setup settled the branch is a no-op. Observed, - /// A retry timer for a row in `Error` ([`retry`]). It acts only while - /// the row still says so and Blox reports the workstation running: it - /// never starts or resumes one. + /// A retry timer for a row in `Error` ([`retry`]), which fires it only + /// while Blox reports the workstation running: it never starts or + /// resumes one. Nobody asked for it, so it is quiet. The row stays + /// `Error` with its reason on show until the checkout has passed. Retry, /// An agent or pipeline start that read the clone of a `Running` branch /// and found it off the branch (see [`verify`]). The only trigger that @@ -145,77 +148,96 @@ pub(crate) async fn ensure_branch_set_up( trigger: SetupTrigger, ) -> Result<(), String> { let host = blox_host(app_handle, store); - run_guarded( - SetupGuard::global(), - store, - &host, - branch_id, - SetupRequest::new(trigger), - Patience::ToTheEnd, - ) - .await - .map(|_| ()) + let request = SetupRequest::new(trigger); + run_guarded(SetupGuard::global(), store, &host, branch_id, request) + .await + .map(|_| ()) } -/// The `start_workspace` command and web arm: ask for `branch_id` to be set -/// up and return the status its row holds. -/// -/// `running` once the branch is checked out and read to be so. `starting` -/// when the run is waiting for the workstation to boot: the call returns -/// rather than hold a caller (and one of the frontend's few setup slots) for -/// the minutes that can take. The run goes on, its outcome lands on the row, -/// and a failure's reason is announced ([`failures`]). An error is the -/// failure that left the row in `Error`. -pub(crate) async fn request_branch_setup( +/// The `start_workspace` command and web arm ([`request_setup`]). +pub(crate) fn request_branch_setup( app_handle: &tauri::AppHandle, store: &Arc, branch_id: &str, -) -> Result { +) -> Result<(), String> { let host = blox_host(app_handle, store); - let status = run_guarded( - SetupGuard::global(), - store, - &host, - branch_id, - SetupRequest::new(SetupTrigger::Requested), - Patience::WhileWorking, - ) - .await?; - let workspace = store + request_setup(SetupGuard::global(), store, &host, branch_id) +} + +/// Ask for `branch_id` to be set up and return without waiting for it. +/// +/// The run's outcome lands on the row, and a failure's reason is announced +/// ([`failures`]): that is all a client is told, and all it needs. An error +/// here means the request itself could not be made. +/// +/// A request for a row that isn't `Starting` is the error view's Retry. The +/// row says `Starting` and its reason is gone before this returns, so the +/// card that asked turns at once. A request that finds a `Starting` row with +/// no run in flight is a new branch's. Both are the user's ask, and a +/// failure of the run is announced as theirs. A request for a `Starting` row +/// whose run is in flight (the startup pass's, a poll's) only joins it. +pub(crate) fn request_setup( + guard: &Arc, + store: &Arc, + host: &Arc, + branch_id: &str, +) -> Result<(), String> { + let branch = store .get_branch(branch_id) - .ok() - .flatten() - .and_then(|branch| branch.workspace_name); - Ok(PollWorkspaceResult { - status: status - .unwrap_or(WorkspaceStatus::Starting) - .as_str() - .to_string(), - workstation_id: workspace.and_then(|name| branches::cached_workstation_id(&name)), - }) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + if !wants_setup(&branch, SetupTrigger::Requested) { + return Ok(()); + } + let workspace = branch + .workspace_name + .as_deref() + .ok_or("Branch has no workspace name")?; + crate::project_transfer::ensure_branch_not_transferring(branch_id)?; + + let was_starting = branch.workspace_status == Some(WorkspaceStatus::Starting); + if !was_starting || !guard.is_in_flight(branch_id) { + guard.user_asked(branch_id); + } + if !was_starting { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) + .map_err(|e| e.to_string())?; + host.failure_changed(branch_id, None, false); + } + let request = SetupRequest::new(SetupTrigger::Requested); + start_run(guard, store, host, workspace, branch_id, request); + Ok(()) } /// Trigger setup for a branch seen in `Starting`, without waiting for it. /// The outcome lands on the row: `Running`, or `Error` with its reason -/// announced. +/// announced. `by_user` says the user brought this about (a resume), so a +/// failure is announced as theirs. pub(crate) fn trigger_branch_setup( app_handle: &tauri::AppHandle, store: &Arc, branch_id: &str, + by_user: bool, ) { + let guard = SetupGuard::global(); + if by_user { + guard.user_asked(branch_id); + } // A poll every 3 s would otherwise spawn a task per tick just to join. - if SetupGuard::global().is_in_flight(branch_id) { + if guard.is_in_flight(branch_id) { return; } - let app_handle = app_handle.clone(); + let host = blox_host(app_handle, store); let store = Arc::clone(store); let branch_id = branch_id.to_string(); tauri::async_runtime::spawn(async move { - if let Err(e) = - ensure_branch_set_up(&app_handle, &store, &branch_id, SetupTrigger::Observed).await - { + let request = SetupRequest::new(SetupTrigger::Observed); + if let Err(e) = run_guarded(guard, &store, &host, &branch_id, request).await { log::warn!("[workspace_setup] setup for branch {branch_id} failed: {e}"); } + // No run, no failure to announce as the user's. + guard.take_user_asked(&branch_id); }); } @@ -242,7 +264,8 @@ pub(crate) fn branch_failed( branch_id: &str, reason: &str, ) { - failures::set(app_handle, branch_id, Some(reason)); + let user_asked = SetupGuard::global().take_user_asked(branch_id); + failures::set(app_handle, store, branch_id, Some(reason), user_asked); arm_retry(app_handle, store, branch_id); } @@ -272,7 +295,9 @@ pub(crate) fn resume_pending_setups( ); for branch in pending { match startup_action(&branch) { - Some(StartupAction::SetUp) => trigger_branch_setup(app_handle, store, &branch.id), + Some(StartupAction::SetUp) => { + trigger_branch_setup(app_handle, store, &branch.id, false) + } Some(StartupAction::ArmRetry) => arm_retry(app_handle, store, &branch.id), None => {} } @@ -317,37 +342,38 @@ pub(crate) fn branches_awaiting_setup(store: &Store) -> Result, + store: &Arc, + host: &Arc, + workspace: &str, + branch_id: &str, + request: SetupRequest, +) -> guard::Run { + let (guard_for_run, store, host) = (Arc::clone(guard), Arc::clone(store), Arc::clone(host)); + let id = branch_id.to_string(); + guard.start(workspace, branch_id, move || async move { + set_up_branch(&guard_for_run, &store, host.as_ref(), &id, &request).await + }) +} -/// [`set_up_branch`] under `guard`: joined per branch, serialized per -/// workstation. Returns the status the branch's row holds afterwards, or -/// `Starting` for a caller that left the run waiting -/// ([`Patience::WhileWorking`]). +/// [`set_up_branch`] under `guard`, waited for. /// /// A caller gets an answer to what it asked. The run it joins may have been -/// started for another reason: a `Requested` call can land on an `Observed` -/// run that found the row no longer `Starting`, or on a `Retry` that found -/// the workstation down, and those end with nothing done. Such an end is -/// not this caller's: it looks at the row again and, if its own trigger -/// still has something to do, runs it. +/// started for another reason and ended with nothing done: an `Observed` run +/// that found the row no longer `Starting`, a repair whose clone read right. +/// Such an end is not this caller's: it looks at the row once more and, if +/// its own trigger still has something to do, runs it. pub(crate) async fn run_guarded( guard: &Arc, store: &Arc, host: &Arc, branch_id: &str, request: SetupRequest, - patience: Patience, -) -> Result, String> { - let status_now = || -> Result, String> { - Ok(store - .get_branch(branch_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Branch not found: {branch_id}"))? - .workspace_status) - }; - for _ in 0..MAX_JOINED_ROUNDS { +) -> Result { + for _ in 0..2 { let branch = store .get_branch(branch_id) .map_err(|e| e.to_string())? @@ -355,69 +381,20 @@ pub(crate) async fn run_guarded( // Checked before taking a slot in the guard, so a trigger with // nothing to do never becomes the run a real request joins. if !wants_setup(&branch, request.trigger) { - return Ok(branch.workspace_status); + break; } let workspace = branch .workspace_name - .clone() + .as_deref() .ok_or("Branch has no workspace name")?; - - // A request can be answered before its run has had its turn, when - // the workstation is busy with another branch - // ([`Patience::WhileWorking`]). The row says `Starting` from now, - // not from whenever the turn comes: a window told `starting` and - // then polling a row that still read `Error` or `Stopped` would stop - // watching a branch that is being set up. Nothing awaits between - // this write and the run being in flight, and a `Starting` row is - // one every recovery path picks up. - if request.trigger == SetupTrigger::Requested - && branch.workspace_status != Some(WorkspaceStatus::Starting) - { - crate::project_transfer::ensure_branch_not_transferring(branch_id)?; - store - .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) - .map_err(|e| e.to_string())?; - } - - let end = { - let guard_for_run = Arc::clone(guard); - let store = Arc::clone(store); - let host = Arc::clone(host); - let id = branch_id.to_string(); - let request = request.clone(); - guard - .run( - &workspace, - branch_id, - patience, - move |progress| async move { - set_up_branch( - &guard_for_run, - &store, - host.as_ref(), - &id, - &request, - &progress, - ) - .await - }, - ) - .await - }; - match end.result { - // Left with the run still waiting, for the workstation or for - // its turn on it. A run in flight is a branch that is starting, - // whether or not its turn has come and the row says so yet. - None => return Ok(Some(WorkspaceStatus::Starting)), - Some(Err(e)) => return Err(e), - Some(Ok(SetupOutcome::SetUp)) => return status_now(), - // This caller's own run: nothing to do is its answer. - Some(Ok(SetupOutcome::NothingToDo)) if !end.joined => return status_now(), - // Someone else's run had nothing to do. Ask again. - Some(Ok(SetupOutcome::NothingToDo)) => {} + let run = start_run(guard, store, host, workspace, branch_id, request.clone()); + let joined = run.joined; + match run.end().await? { + SetupOutcome::NothingToDo if joined => {} + outcome => return Ok(outcome), } } - status_now() + Ok(SetupOutcome::NothingToDo) } /// Whether `trigger` has anything to do for a branch in this state. A @@ -449,23 +426,22 @@ fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { /// `Running` is written only over a clone that reads as on the branch. /// /// Every failure past the first lookups is returned, and leaves the row in -/// `Error` with its reason recorded and a retry timer set. One exception: a -/// repair of a `Running` branch that was refused (a worktree with changes in -/// the way, commits that would be left behind, a live session in the clone) -/// puts the row back to `Running`. The branch was usable before the session -/// that asked came along and is exactly as usable now; only that session -/// fails, with the reason. +/// `Error` with its reason recorded and, unless the checkout refused over +/// what is in the clone, a retry timer set. One exception: a repair of a +/// `Running` branch that was refused (a worktree with changes in the way, +/// commits that would be left behind, a live session in the clone) puts the +/// row back to `Running`. The branch was usable before the session that +/// asked came along and is exactly as usable now; only that session fails, +/// with the reason. /// -/// Callers go through [`run_guarded`]. +/// Callers go through [`run_guarded`] or [`request_setup`]. pub(crate) async fn set_up_branch( guard: &Arc, store: &Arc, host: &dyn SetupHost, branch_id: &str, request: &SetupRequest, - progress: &Progress, ) -> Result { - let trigger = request.trigger; // A transfer provisions its own branches; refusing leaves the row alone. crate::project_transfer::ensure_branch_not_transferring(branch_id)?; @@ -473,6 +449,12 @@ pub(crate) async fn set_up_branch( .get_branch(branch_id) .map_err(|e| e.to_string())? .ok_or_else(|| format!("Branch not found: {branch_id}"))?; + // A retry that finds the row `Starting` was joined, while it waited for + // its turn, by the user's Retry or a resume. It is that run now. + let trigger = match (request.trigger, &branch.workspace_status) { + (SetupTrigger::Retry, Some(WorkspaceStatus::Starting)) => SetupTrigger::Observed, + (trigger, _) => trigger, + }; // Read again now that the workstation's turn has come: the run ahead of // this one may have settled the branch. if !wants_setup(&branch, trigger) { @@ -480,19 +462,6 @@ pub(crate) async fn set_up_branch( } let was_running = branch.workspace_status == Some(WorkspaceStatus::Running); - // A retry goes ahead only on a workstation Blox reports running. One - // that is down, or can't be asked about, is looked at again later. - if trigger == SetupTrigger::Retry { - let reported = match branch.workspace_name.as_deref() { - Some(workspace) => host.workspace_status(workspace).await.ok().flatten(), - None => None, - }; - if !reports_running(reported.as_deref()) { - retry::schedule(guard, host, branch_id); - return Ok(SetupOutcome::NothingToDo); - } - } - // A repair was asked for by a session that read the clone off the // branch. The run ahead of this one (another session's repair, a // sibling branch's setup) may have changed that; when the clone reads @@ -510,38 +479,35 @@ pub(crate) async fn set_up_branch( } } - // A request for a branch that had settled somewhere else (a failed - // setup, a stale `starting` in the window that asked, a repair of a - // `Running` one) puts it back in `Starting`, where the pollers hold it + // A run someone or something asked for puts a branch that had settled + // somewhere else (a stale `starting` in the window that asked, a repair + // of a `Running` one) back in `Starting`, where the pollers hold it // rather than promote it under this run and the session queue holds - // whatever else would start on the branch. - if branch.workspace_status != Some(WorkspaceStatus::Starting) { - store - .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) - .map_err(|e| e.to_string())?; + // whatever else would start on the branch. A retry on its own does + // neither: the card would flip to a spinner and lose its reason for an + // attempt nobody is watching. `Error` holds the queue as well. + if trigger != SetupTrigger::Retry { + if branch.workspace_status != Some(WorkspaceStatus::Starting) { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Starting) + .map_err(|e| e.to_string())?; + } + host.failure_changed(branch_id, None, false); } - host.failure_changed(branch_id, None); // A repair of a row that is not `Running` is the setup that row was // waiting for, and provisions like one. let may_start_workstation = trigger.may_start_workstation() || (trigger == SetupTrigger::Repair && !was_running); - let provisioned = provision_and_check_out( - guard, - store, - host, - &branch, - request, - may_start_workstation, - progress, - ) - .await; + let provisioned = + provision_and_check_out(guard, store, host, &branch, request, may_start_workstation).await; let error = match provisioned { Ok(()) => { let changed = store .update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) .map_err(|e| e.to_string())?; host.status_written(branch_id, changed, &WorkspaceStatus::Running); + host.failure_changed(branch_id, None, false); guard.forget_retries(branch_id); log::info!( "[workspace_setup] branch={branch_id} checked out '{}' and running", @@ -553,15 +519,14 @@ pub(crate) async fn set_up_branch( }; log::warn!("[workspace_setup] branch={branch_id} setup failed: {error}"); - // Only over `Starting`. A poll that saw the workstation suspended or - // gone while this ran has already said something truer than `Error`, - // and with a way forward (Resume). - let still_starting = matches!( - store.get_branch(branch_id), - Ok(Some(row)) if row.workspace_status == Some(WorkspaceStatus::Starting) - ); - if still_starting { - if trigger == SetupTrigger::Repair && was_running && error.is_refusal() { + // Only over `Starting`, or the `Error` a retry left in place. A poll + // that saw the workstation suspended or gone while this ran has already + // said something truer than `Error`, and with a way forward (Resume). + let row = store.get_branch(branch_id).ok().flatten(); + match row.and_then(|row| row.workspace_status) { + Some(WorkspaceStatus::Starting) + if trigger == SetupTrigger::Repair && was_running && error.is_refusal() => + { // Nothing was touched, so the branch is what it was. The write // kicks the session queue, which held while the row said // `Starting`. @@ -570,13 +535,23 @@ pub(crate) async fn set_up_branch( { host.status_written(branch_id, changed, &WorkspaceStatus::Running); } - } else { - store - .update_branch_workspace_status(branch_id, &WorkspaceStatus::Error) - .ok(); - host.failure_changed(branch_id, Some(error.message())); - retry::schedule(guard, host, branch_id); } + Some(status @ (WorkspaceStatus::Starting | WorkspaceStatus::Error)) => { + if status == WorkspaceStatus::Starting { + store + .update_branch_workspace_status(branch_id, &WorkspaceStatus::Error) + .ok(); + } + let user_asked = guard.take_user_asked(branch_id); + host.failure_changed(branch_id, Some(error.message()), user_asked); + if error.worth_retrying() { + retry::schedule(guard, host, branch_id); + } else { + // Left to the error view's Retry. + guard.forget_retries(branch_id); + } + } + _ => {} } Err(error.message().to_string()) } @@ -588,7 +563,6 @@ async fn provision_and_check_out( branch: &store::Branch, request: &SetupRequest, may_start_workstation: bool, - progress: &Progress, ) -> Result<(), CheckoutError> { let failed = CheckoutError::Failed; let checkout = checkout_for(store, branch).map_err(failed)?; @@ -611,25 +585,18 @@ async fn provision_and_check_out( "[workspace_setup] branch={} workspace={workspace} is already booting, waiting for it", branch.id ); - wait_until_running( - store, - host, - branch, - workspace, - "it was still booting", - progress, - ) - .await - .map_err(failed)?; + wait_until_running(store, host, branch, workspace, "it was still booting") + .await + .map_err(failed)?; } else if may_start_workstation { - start_workstation(store, host, branch, &checkout, progress) + start_workstation(store, host, branch, &checkout) .await .map_err(failed)?; } else { // A repair or a retry found the workstation down. The clone is as // it was, and what the row should say about a workstation that // isn't running is the pollers' to settle. - return Err(CheckoutError::Refused(format!( + return Err(CheckoutError::Blocked(format!( "Workspace '{workspace}' is not running (Blox reports it as {}), so branch '{}' \ could not be checked out in it", reported.as_deref().unwrap_or("unknown"), diff --git a/apps/staged/src-tauri/src/workspace_setup/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/checkout.rs index 71a45b56e..105aa00e2 100644 --- a/apps/staged/src-tauri/src/workspace_setup/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/checkout.rs @@ -28,18 +28,32 @@ pub(crate) enum CheckoutError { /// clone is as it was; trying again changes nothing until someone has /// dealt with what is in it. Refused(String), + /// Something outside the clone is in the way for now: a session working + /// in it, a workstation that is not running. The clone is as it was, + /// and trying again later is worth it. + Blocked(String), /// A step failed: a fetch, a probe, the checkout itself. Failed(String), } impl CheckoutError { + /// Whether the clone was left exactly as it was. pub(crate) fn is_refusal(&self) -> bool { - matches!(self, CheckoutError::Refused(_)) + !matches!(self, CheckoutError::Failed(_)) + } + + /// Whether trying again on its own could end differently. Not after a + /// refusal over what is in the clone: that waits for someone to deal + /// with it and ask again. + pub(crate) fn worth_retrying(&self) -> bool { + !matches!(self, CheckoutError::Refused(_)) } pub(crate) fn message(&self) -> &str { match self { - CheckoutError::Refused(message) | CheckoutError::Failed(message) => message, + CheckoutError::Refused(message) + | CheckoutError::Blocked(message) + | CheckoutError::Failed(message) => message, } } @@ -47,6 +61,7 @@ impl CheckoutError { pub(crate) fn map_message(self, f: impl FnOnce(String) -> String) -> Self { match self { CheckoutError::Refused(message) => CheckoutError::Refused(f(message)), + CheckoutError::Blocked(message) => CheckoutError::Blocked(f(message)), CheckoutError::Failed(message) => CheckoutError::Failed(f(message)), } } diff --git a/apps/staged/src-tauri/src/workspace_setup/clone.rs b/apps/staged/src-tauri/src/workspace_setup/clone.rs index 55a7ef4b8..97be10662 100644 --- a/apps/staged/src-tauri/src/workspace_setup/clone.rs +++ b/apps/staged/src-tauri/src/workspace_setup/clone.rs @@ -98,7 +98,7 @@ pub(super) async fn check_out_in_clone( let (workspace, expected) = (&checkout.workspace, &checkout.branch_name); return match verify::read_head(host, checkout).await { Ok(head) if verify::is_on_branch(&head, checkout) => Ok(()), - Ok(head) => Err(CheckoutError::Refused(format!( + Ok(head) => Err(CheckoutError::Blocked(format!( "Workspace '{workspace}' has {head} checked out, where branch '{expected}' was \ expected. It was not switched because {live}; try again once that session has \ finished." diff --git a/apps/staged/src-tauri/src/workspace_setup/failures.rs b/apps/staged/src-tauri/src/workspace_setup/failures.rs index 7dee3d990..14cb19504 100644 --- a/apps/staged/src-tauri/src/workspace_setup/failures.rs +++ b/apps/staged/src-tauri/src/workspace_setup/failures.rs @@ -5,10 +5,17 @@ //! was joined to — the startup pass, a poll's trigger, a retry — failed into //! the log, and the branch showed "Something went wrong." //! -//! So the reason is kept here and announced: every change is emitted as -//! `workspace-setup-error` (`{ branchId, error }`, `error: null` when it is -//! cleared), and [`get_workspace_setup_errors`] hands a window that mounts -//! later what it missed. +//! So the reason is kept here and announced: a change is emitted as +//! `workspace-setup-error` (`{ branchId, error, userRequested }`, `error: +//! null` when it is cleared), and [`get_workspace_setup_errors`] hands a +//! window that mounts later what it missed. This is how a failure reaches +//! the UI; no client call waits on a setup to be told. +//! +//! `userRequested` says the failed setup was one the user asked for (a new +//! branch, the error view's Retry, a resume), which is what a client raises +//! a toast for. A retry on a timer, the startup pass and a poll's own run +//! fail into the card only, and a reason that has not changed is not +//! announced again. //! //! In memory, per process: the branch row keeps only its status. After a //! restart the reasons are gone until the next attempt fails, which the @@ -19,6 +26,8 @@ use std::sync::{Mutex, OnceLock}; use serde::Serialize; +use crate::store::{Store, WorkspaceStatus}; + /// The event a reason's change is announced with. pub(crate) const EVENT: &str = "workspace-setup-error"; @@ -27,6 +36,7 @@ pub(crate) const EVENT: &str = "workspace-setup-error"; struct SetupErrorEvent<'a> { branch_id: &'a str, error: Option<&'a str>, + user_requested: bool, } fn reasons() -> &'static Mutex> { @@ -35,31 +45,59 @@ fn reasons() -> &'static Mutex> { } /// Record why `branch_id` failed (`Some`) or that it no longer has a reason -/// (`None`), and tell every window and web client. Clearing a branch that -/// has no reason announces nothing. +/// (`None`), and tell every window and web client when that changed +/// something, or when the failure is of a setup the user asked for. +/// +/// Recording a failure is also when the reasons of branches that are gone, +/// or no longer in `Error`, are dropped: a deleted branch, or one a poll +/// moved on, would otherwise keep its reason for every window to come. pub(crate) fn set( app_handle: &tauri::AppHandle, + store: &Store, branch_id: &str, error: Option<&str>, + user_requested: bool, ) { - let changed = { - let mut reasons = reasons().lock().unwrap_or_else(|p| p.into_inner()); - match error { - Some(error) => { - reasons.insert(branch_id.to_string(), error.to_string()); - true - } - None => reasons.remove(branch_id).is_some(), + let emit = |branch_id: &str, error: Option<&str>| { + let event = SetupErrorEvent { + branch_id, + error, + user_requested: user_requested && error.is_some(), + }; + crate::web_server::emit_to_all(app_handle, EVENT, event); + }; + let Some(error) = error else { + if lock().remove(branch_id).is_some() { + emit(branch_id, None); } + return; }; - if changed { - crate::web_server::emit_to_all(app_handle, EVENT, SetupErrorEvent { branch_id, error }); + let previous = lock().insert(branch_id.to_string(), error.to_string()); + if user_requested || previous.as_deref() != Some(error) { + emit(branch_id, Some(error)); + } + let recorded: Vec = lock().keys().cloned().collect(); + for stale in recorded.iter().filter(|id| !is_in_error(store, id)) { + if lock().remove(stale).is_some() { + emit(stale, None); + } } } +fn lock() -> std::sync::MutexGuard<'static, HashMap> { + reasons().lock().unwrap_or_else(|p| p.into_inner()) +} + +fn is_in_error(store: &Store, branch_id: &str) -> bool { + matches!( + store.get_branch(branch_id), + Ok(Some(branch)) if branch.workspace_status == Some(WorkspaceStatus::Error) + ) +} + /// Every branch's recorded reason. pub(crate) fn all() -> HashMap { - reasons().lock().unwrap_or_else(|p| p.into_inner()).clone() + lock().clone() } /// The reasons the branches in `Error` failed for, by branch id, for a diff --git a/apps/staged/src-tauri/src/workspace_setup/guard.rs b/apps/staged/src-tauri/src/workspace_setup/guard.rs index e7fdaa438..a8374a9aa 100644 --- a/apps/staged/src-tauri/src/workspace_setup/guard.rs +++ b/apps/staged/src-tauri/src/workspace_setup/guard.rs @@ -9,25 +9,26 @@ //! //! So a run is keyed by branch: a second caller for a branch whose setup is //! in flight gets that run's result rather than starting its own, and is -//! told that it joined ([`RunEnd::joined`]), since the run it joined was +//! told that it joined ([`Run::joined`]), since the run it joined was //! started for someone else's reason. Runs for different branches of one //! workstation take turns, since they may share a clone. //! //! A run is its own task. A caller that goes away — a dropped web request, a //! closed window — only stops waiting; the setup still finishes and still -//! writes the branch's status. A caller can also choose to stop waiting -//! while the run itself is only waiting, for its turn or for a workstation -//! to boot ([`Patience::WhileWorking`]). +//! writes the branch's status. Most callers never wait at all +//! ([`SetupGuard::start`]): the outcome lands on the row. //! -//! The guard holds two more things that are per process for the same +//! The guard holds three more things that are per process for the same //! reason the runs are: //! //! - the gate of each clone ([`SetupGuard::clone_gate`]), which a setup //! holds over its checkout and an agent start holds from reading the //! clone until its session is live, so neither happens under the other; -//! - each failed branch's retry count and timer ticket ([`super::retry`]). +//! - each failed branch's retry count and timer ticket ([`super::retry`]); +//! - which branches have a setup the user asked for still to end, so its +//! failure is announced as one ([`SetupGuard::user_asked`]). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::future::Future; use std::sync::{Arc, Mutex, OnceLock}; @@ -35,55 +36,27 @@ use tokio::sync::{watch, OwnedMutexGuard}; use super::SetupOutcome; -/// Where a run is. -#[derive(Debug, Clone, PartialEq)] -enum RunState { - /// Behind another branch's run on the same workstation, which may itself - /// be waiting for the workstation. - Queued, - /// Doing something that ends on its own time: an exec, a checkout. - Working, - /// Waiting for a workstation to report running, for minutes at worst. - WaitingForWorkstation, - Done(Result), -} - -/// How long a caller stays with a run. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Patience { - /// Until the run has a result. - ToTheEnd, - /// Until the run has a result or is only waiting — for its turn behind - /// another branch's run, or for a workstation to boot — whichever comes - /// first. The run goes on either way. - WhileWorking, -} +/// How a run ended. `None` in its channel while it is still going. +type RunResult = Result; -/// What a caller of [`SetupGuard::run`] came back with. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct RunEnd { +/// A run a caller started or joined ([`SetupGuard::start`]). +pub(crate) struct Run { /// Whether the run was already in flight, started by another caller for /// its own reason. pub(crate) joined: bool, - /// The run's result, or `None` when the caller left it waiting - /// ([`Patience::WhileWorking`]). - pub(crate) result: Option>, -} - -/// A run's way of saying what it is doing, for callers deciding whether to -/// stay ([`Patience`]). -#[derive(Clone)] -pub(crate) struct Progress { - state: Arc>, + result: watch::Receiver>, } -impl Progress { - pub(crate) fn waiting_for_workstation(&self) { - self.state.send_replace(RunState::WaitingForWorkstation); - } - - pub(crate) fn working(&self) { - self.state.send_replace(RunState::Working); +impl Run { + /// Wait for the run's result. + pub(crate) async fn end(mut self) -> RunResult { + // The sender is dropped without a result only if the run panicked; + // its entry is gone with it. + let lost = || Err("Workspace setup ended without a result".to_string()); + match self.result.wait_for(Option::is_some).await { + Ok(result) => result.clone().unwrap_or_else(lost), + Err(_) => lost(), + } } } @@ -104,14 +77,16 @@ type CloneKey = (String, Option); #[derive(Default)] pub(crate) struct SetupGuard { - /// Branch id → the state of the run in flight for it. - in_flight: Mutex>>, + /// Branch id → the result of the run in flight for it. + in_flight: Mutex>>>, /// Workspace name → the lock its runs take turns on. workspace_locks: Mutex>>>, /// Clone → its gate. clone_gates: Mutex>>>, /// Branch id → its retry schedule. retries: Mutex>, + /// Branches with a setup the user asked for still to end. + user_asked: Mutex>, } impl SetupGuard { @@ -126,96 +101,62 @@ impl SetupGuard { self.lock_in_flight().contains_key(branch_id) } - /// Run `setup` for `branch_id` on `workspace`, or wait for the run - /// already in flight for that branch, and return how it ended. + /// Start `setup` for `branch_id` on `workspace` as its own task, or + /// join the run already in flight for that branch. Returns at once; + /// [`Run::end`] waits for the result. /// - /// `setup` is not called when the caller joins. - pub(crate) async fn run( - self: &Arc, - workspace: &str, - branch_id: &str, - patience: Patience, - setup: F, - ) -> RunEnd + /// `setup` is not called when the caller joins. It runs once the + /// workstation's turn comes. + pub(crate) fn start(self: &Arc, workspace: &str, branch_id: &str, setup: F) -> Run where - F: FnOnce(Progress) -> Fut + Send + 'static, - Fut: Future> + Send + 'static, + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + Send + 'static, { - let (mut state, joined) = { - let mut in_flight = self.lock_in_flight(); - match in_flight.get(branch_id) { - Some(state) => (state.clone(), true), - None => { - let turn = self.workspace_lock(workspace); - // Taken here when it is free, so a run that starts at - // once is never seen as queued. - let taken = Arc::clone(&turn).try_lock_owned().ok(); - let (tx, rx) = watch::channel(match taken { - Some(_) => RunState::Working, - None => RunState::Queued, - }); - let tx = Arc::new(tx); - in_flight.insert(branch_id.to_string(), rx.clone()); - let entry = InFlight { - guard: Arc::clone(self), - branch_id: branch_id.to_string(), - }; - let progress = Progress { - state: Arc::clone(&tx), - }; - tauri::async_runtime::spawn(async move { - let result = { - let _turn = match taken { - Some(turn) => turn, - None => { - let turn = turn.lock_owned().await; - progress.working(); - turn - } - }; - setup(progress).await - }; - // Cleared before the result is published: a caller - // that sees the result must be able to start a fresh - // run, not join a finished one. - drop(entry); - tx.send_replace(RunState::Done(result)); - }); - (rx, false) - } - } + let mut in_flight = self.lock_in_flight(); + if let Some(result) = in_flight.get(branch_id) { + return Run { + joined: true, + result: result.clone(), + }; + } + let (tx, rx) = watch::channel(None); + in_flight.insert(branch_id.to_string(), rx.clone()); + let turn = self.workspace_lock(workspace); + let entry = InFlight { + guard: Arc::clone(self), + branch_id: branch_id.to_string(), }; - - loop { - let seen = state.borrow_and_update().clone(); - match seen { - RunState::Done(result) => { - return RunEnd { - joined, - result: Some(result), - } - } - RunState::Queued | RunState::WaitingForWorkstation - if patience == Patience::WhileWorking => - { - return RunEnd { - joined, - result: None, - } - } - RunState::Queued | RunState::Working | RunState::WaitingForWorkstation => {} - } - // The sender is dropped without a result only if the run - // panicked; its entry is gone with it. - if state.changed().await.is_err() { - return RunEnd { - joined, - result: Some(Err("Workspace setup ended without a result".to_string())), - }; - } + tauri::async_runtime::spawn(async move { + let result = { + let _turn = turn.lock_owned().await; + setup().await + }; + // Cleared before the result is published: a caller that sees + // the result must be able to start a fresh run, not join a + // finished one. + drop(entry); + tx.send_replace(Some(result)); + }); + Run { + joined: false, + result: rx, } } + /// The user asked for `branch_id` to be set up (a new branch, the error + /// view's Retry, a resume). The next failure announced for it is + /// announced as theirs, which is what raises a toast + /// ([`super::failures`]). Forgotten when the run in flight ends. + pub(crate) fn user_asked(&self, branch_id: &str) { + self.lock_user_asked().insert(branch_id.to_string()); + } + + /// Whether the user asked for the setup of `branch_id` that is ending, + /// asked once: a later failure is not theirs. + pub(crate) fn take_user_asked(&self, branch_id: &str) -> bool { + self.lock_user_asked().remove(branch_id) + } + /// Take the gate of the clone at `clone_dir` on `workspace`. /// /// Held by a setup from before it looks for live sessions in the clone @@ -284,9 +225,13 @@ impl SetupGuard { self.retries.lock().unwrap_or_else(|p| p.into_inner()) } + fn lock_user_asked(&self) -> std::sync::MutexGuard<'_, HashSet> { + self.user_asked.lock().unwrap_or_else(|p| p.into_inner()) + } + fn lock_in_flight( &self, - ) -> std::sync::MutexGuard<'_, HashMap>> { + ) -> std::sync::MutexGuard<'_, HashMap>>> { self.in_flight.lock().unwrap_or_else(|p| p.into_inner()) } @@ -316,5 +261,7 @@ struct InFlight { impl Drop for InFlight { fn drop(&mut self) { self.guard.lock_in_flight().remove(&self.branch_id); + // A run that ended without announcing a failure used nobody's ask. + self.guard.take_user_asked(&self.branch_id); } } diff --git a/apps/staged/src-tauri/src/workspace_setup/host.rs b/apps/staged/src-tauri/src/workspace_setup/host.rs index fb302d50f..f7ba8f027 100644 --- a/apps/staged/src-tauri/src/workspace_setup/host.rs +++ b/apps/staged/src-tauri/src/workspace_setup/host.rs @@ -100,9 +100,10 @@ pub(crate) trait SetupHost: Send + Sync + 'static { /// for a `Running` write (see [`branches::workspace_status_written`]). fn status_written(&self, branch_id: &str, changed: bool, status: &WorkspaceStatus); - /// The reason a branch is in `Error` changed: `Some` when a run failed - /// it, `None` when a new run began ([`failures`]). - fn failure_changed(&self, branch_id: &str, error: Option<&str>); + /// The reason a branch is in `Error`: `Some` when a run failed it, with + /// whether the user had asked for that run, `None` when it has none any + /// more ([`failures`]). + fn failure_changed(&self, branch_id: &str, error: Option<&str>, user_asked: bool); /// Call [`retry::fire`] for `branch_id` with `ticket` once `delay` has /// passed. @@ -212,8 +213,8 @@ impl SetupHost for BloxHost { branches::workspace_status_written(&self.app_handle, branch_id, changed, status); } - fn failure_changed(&self, branch_id: &str, error: Option<&str>) { - failures::set(&self.app_handle, branch_id, error); + fn failure_changed(&self, branch_id: &str, error: Option<&str>, user_asked: bool) { + failures::set(&self.app_handle, &self.store, branch_id, error, user_asked); } fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { diff --git a/apps/staged/src-tauri/src/workspace_setup/poll.rs b/apps/staged/src-tauri/src/workspace_setup/poll.rs index 95d772d8a..bc06f59ad 100644 --- a/apps/staged/src-tauri/src/workspace_setup/poll.rs +++ b/apps/staged/src-tauri/src/workspace_setup/poll.rs @@ -418,7 +418,7 @@ fn app_effects<'a>(store: &'a Arc, app_handle: &'a AppHandle) -> PollEffe branches::workspace_status_written(app_handle, branch_id, changed, status); }), trigger_setup: Box::new(move |branch_id| { - super::trigger_branch_setup(app_handle, store, branch_id); + super::trigger_branch_setup(app_handle, store, branch_id, false); }), arm_retry: Box::new(move |branch_id| { super::arm_retry(app_handle, store, branch_id); diff --git a/apps/staged/src-tauri/src/workspace_setup/retry.rs b/apps/staged/src-tauri/src/workspace_setup/retry.rs index 0b75b2a26..ed21946d3 100644 --- a/apps/staged/src-tauri/src/workspace_setup/retry.rs +++ b/apps/staged/src-tauri/src/workspace_setup/retry.rs @@ -4,27 +4,40 @@ //! pass skipped it and the pollers held it. A dropped exec during a fetch //! was enough to leave a branch there until the user deleted it. //! -//! Every write of `Error` is now followed by a retry timer, backed off from -//! a minute to half an hour ([`RetryPolicy`]). When it fires and the row is -//! still `Error`, setup runs with [`SetupTrigger::Retry`]: it looks at the -//! workstation first and goes ahead only when Blox reports it running. A -//! retry never starts or resumes a workstation — that is the user's call, -//! through the error view's Retry or the project's Resume — so a branch -//! whose workstation is down stays `Error` and the timer is set again. +//! A write of `Error` is now followed by a retry timer, backed off from a +//! minute to half an hour and spread by some jitter ([`RetryPolicy`]), so +//! the branches a restart finds failed are not all retried in one instant. +//! When it fires and the row is still `Error`, it looks at the workstation +//! first and runs setup with [`SetupTrigger::Retry`] only when Blox reports +//! it running. A retry never starts or resumes a workstation — that is the +//! user's call, through the error view's Retry or the project's Resume — so +//! a branch whose workstation is down stays `Error` and the timer is set +//! again. +//! +//! A retry is quiet. Nobody asked for it, so the row stays `Error` and +//! keeps its reason while it runs, and a failure that says what the last +//! one said announces nothing. +//! +//! One failure sets no timer: a checkout refused over what is in the clone +//! ([`CheckoutError::worth_retrying`]). Trying again changes nothing until +//! someone has dealt with it, and then the error view's Retry is theirs to +//! press. //! //! Timers are armed by setup itself when it fails, by the pollers when they //! see an `Error` row behind a running workstation or write `Error` //! themselves, by a failed resume, and by the startup pass for the rows a //! previous run left in `Error`. A branch has at most one timer that //! counts ([`SetupGuard::retry_fired`]). +//! +//! [`CheckoutError::worth_retrying`]: super::checkout::CheckoutError::worth_retrying use std::sync::Arc; use std::time::Duration; use crate::store::{Store, WorkspaceStatus}; -use super::guard::{Patience, SetupGuard}; -use super::host::SetupHost; +use super::guard::SetupGuard; +use super::host::{reports_running, SetupHost}; use super::{run_guarded, SetupRequest, SetupTrigger}; /// How long to wait before each retry. @@ -33,6 +46,9 @@ pub(crate) struct RetryPolicy { /// The wait before the first retry; it doubles with each further one. pub(crate) base: Duration, pub(crate) cap: Duration, + /// How much longer than its backed-off wait a retry may wait, in + /// percent. + pub(crate) jitter_percent: u64, } impl RetryPolicy { @@ -44,15 +60,19 @@ impl RetryPolicy { pub(crate) const DEFAULT: RetryPolicy = RetryPolicy { base: Duration::from_secs(60), cap: Duration::from_secs(30 * 60), + jitter_percent: 20, }; - /// The wait before retry number `attempt` (0 for the first). - pub(crate) fn delay(&self, attempt: u32) -> Duration { + /// The wait before retry number `attempt` (0 for the first): the + /// backed-off wait, plus the share of its jitter that `random` picks. + pub(crate) fn delay(&self, attempt: u32, random: u64) -> Duration { let doubled = self .base .checked_mul(2u32.saturating_pow(attempt.min(16))) .unwrap_or(self.cap); - doubled.min(self.cap) + let backed_off = doubled.min(self.cap); + let spread = backed_off.as_millis() as u64 * self.jitter_percent / 100; + backed_off + Duration::from_millis(random % (spread + 1)) } } @@ -60,7 +80,7 @@ impl RetryPolicy { /// its last one. pub(crate) fn schedule(guard: &SetupGuard, host: &dyn SetupHost, branch_id: &str) { let (attempt, ticket) = guard.schedule_retry(branch_id); - let delay = host.retry_policy().delay(attempt); + let delay = host.retry_policy().delay(attempt, rand::random()); log::info!( "[workspace_setup] branch={branch_id} retry {} in {}s", attempt + 1, @@ -79,8 +99,9 @@ pub(crate) fn arm(guard: &SetupGuard, host: &dyn SetupHost, branch_id: &str) { schedule(guard, host, branch_id); } -/// A retry timer fired. Runs setup when the timer still counts and the row -/// still says `Error`; anything else has moved the branch on. +/// A retry timer fired. Runs setup when the timer still counts, the row +/// still says `Error` and its workstation is running. A row that is gone or +/// has moved on has nothing to retry and no reason to keep. pub(crate) async fn fire( guard: &Arc, store: &Arc, @@ -92,17 +113,30 @@ pub(crate) async fn fire( return; } let row = store.get_branch(branch_id).ok().flatten(); - if !matches!(&row, Some(branch) if branch.workspace_status == Some(WorkspaceStatus::Error)) { + let Some(branch) = row.filter(|row| row.workspace_status == Some(WorkspaceStatus::Error)) + else { guard.forget_retries(branch_id); + host.failure_changed(branch_id, None, false); return; - } + }; // A transfer provisions its own branches. Look again once it is over. if crate::project_transfer::is_branch_transferring(branch_id) { schedule(guard, host.as_ref(), branch_id); return; } + // Looked at here, before a run exists: a workstation that is down, or + // can't be asked about, is looked at again later, and a request never + // finds itself joined to a retry with nothing to do. + let reported = match branch.workspace_name.as_deref() { + Some(workspace) => host.workspace_status(workspace).await.ok().flatten(), + None => None, + }; + if !reports_running(reported.as_deref()) { + schedule(guard, host.as_ref(), branch_id); + return; + } let request = SetupRequest::new(SetupTrigger::Retry); - if let Err(e) = run_guarded(guard, store, host, branch_id, request, Patience::ToTheEnd).await { + if let Err(e) = run_guarded(guard, store, host, branch_id, request).await { // The run recorded the reason and set the next timer. log::warn!("[workspace_setup] retry of branch {branch_id} failed: {e}"); } diff --git a/apps/staged/src-tauri/src/workspace_setup/tests.rs b/apps/staged/src-tauri/src/workspace_setup/tests.rs index 3ace341e2..5e69332e2 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests.rs @@ -4,9 +4,8 @@ //! same [`branches::GitRunner`] the real checkout runs on. //! //! The rest is in the submodules: what the checkout does to the clone -//! ([`checkout`]), the status pollers ([`poll`]), what a caller of setup -//! gets back ([`requests`]), retries -//! and failure reasons ([`retry`]), the sessions sharing a clone +//! ([`checkout`]), the status pollers ([`poll`]), what asking for setup +//! does ([`requests`]), retries and failure reasons ([`retry`]), the sessions sharing a clone //! ([`sessions`]), the reads of the clone ([`verify`]), and that every //! trigger and check is still called from where it should be ([`wiring`]). @@ -20,8 +19,9 @@ use async_trait::async_trait; use tokio::sync::Semaphore; use super::checkout::{BranchCheckout, CheckoutError}; -use super::guard::{Patience, SetupGuard}; +use super::guard::SetupGuard; use super::host::{Checkout, ReadyWait, SetupHost}; +use super::retry::RetryPolicy; use super::verify::{parse_clone_head, CloneHead, HeadRead, CLONE_HEAD_PROBE_SCRIPT}; use super::{ branches_awaiting_setup, run_guarded, startup_action, SetupOutcome, SetupRequest, SetupTrigger, @@ -33,6 +33,7 @@ use crate::store::{self, Store, WorkspaceStatus}; use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; mod checkout; +mod failures; mod poll; mod requests; mod retry; @@ -168,6 +169,8 @@ struct FakeHost { written: Mutex>, /// Every change of a branch's failure reason, in order. failures: Mutex)>>, + /// The branches whose failure was announced as one the user asked for. + user_asked_failures: Mutex>, /// Every retry timer asked for: branch, delay, ticket. Nothing sleeps; /// a test fires the ones it wants. retries: Mutex>, @@ -194,6 +197,7 @@ impl FakeHost { calls: Mutex::new(Vec::new()), written: Mutex::new(Vec::new()), failures: Mutex::new(Vec::new()), + user_asked_failures: Mutex::new(Vec::new()), retries: Mutex::new(Vec::new()), on_check_disconnected: None, checked_sessions: Mutex::new(Vec::new()), @@ -244,6 +248,10 @@ impl FakeHost { .and_then(|(_, reason)| reason) } + fn user_asked_failures(&self) -> Vec { + self.user_asked_failures.lock().unwrap().clone() + } + fn retries(&self) -> Vec<(String, Duration, u64)> { self.retries.lock().unwrap().clone() } @@ -351,11 +359,17 @@ impl SetupHost for FakeHost { .push((branch_id.to_string(), changed, status.clone())); } - fn failure_changed(&self, branch_id: &str, error: Option<&str>) { + fn failure_changed(&self, branch_id: &str, error: Option<&str>, user_asked: bool) { self.failures .lock() .unwrap() .push((branch_id.to_string(), error.map(str::to_string))); + if user_asked { + self.user_asked_failures + .lock() + .unwrap() + .push(branch_id.to_string()); + } } fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { @@ -392,6 +406,14 @@ impl SetupHost for FakeHost { retry_after: Duration::from_millis(5), } } + + /// No jitter, so a test can say what each wait is. + fn retry_policy(&self) -> RetryPolicy { + RetryPolicy { + jitter_percent: 0, + ..RetryPolicy::DEFAULT + } + } } /// One request for setup, waited for to its end. Returns the status the @@ -414,15 +436,9 @@ async fn set_up_under( trigger: SetupTrigger, ) -> Result, String> { let host: Arc = host.clone(); - run_guarded( - guard, - store, - &host, - &branch.id, - SetupRequest::new(trigger), - Patience::ToTheEnd, - ) - .await + run_guarded(guard, store, &host, &branch.id, SetupRequest::new(trigger)) + .await + .map(|_| status_of(store, branch)) } // ── The setup routine ─────────────────────────────────────────────────────── @@ -798,7 +814,7 @@ async fn concurrent_callers_join_one_setup() { ); tokio::spawn(async move { let request = SetupRequest::new(trigger); - run_guarded(&guard, &store, &host, &id, request, Patience::ToTheEnd).await + run_guarded(&guard, &store, &host, &id, request).await }) }) .collect(); @@ -808,7 +824,7 @@ async fn concurrent_callers_join_one_setup() { // Room for every caller to run its own checkout, were it going to. gate.add_permits(4); for caller in callers { - assert_eq!(caller.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); + assert_eq!(caller.await.unwrap(), Ok(SetupOutcome::SetUp)); } assert_eq!(host.count("check_out"), 1); @@ -842,7 +858,7 @@ async fn joined_callers_share_a_failure() { ); tokio::spawn(async move { let request = SetupRequest::new(SetupTrigger::Requested); - run_guarded(&guard, &store, &host, &id, request, Patience::ToTheEnd).await + run_guarded(&guard, &store, &host, &id, request).await }) }) .collect(); @@ -891,21 +907,13 @@ async fn setups_in_one_workstation_take_turns() { async fn a_panicked_setup_releases_the_branch() { let guard = Arc::new(SetupGuard::default()); - let ended = guard - .run(WORKSPACE, "branch", Patience::ToTheEnd, |_| async { - panic!("setup blew up") - }) - .await; - assert!(matches!(ended.result, Some(Err(_))), "{ended:?}"); - assert!(!ended.joined); + let run = guard.start(WORKSPACE, "branch", || async { panic!("setup blew up") }); + assert!(!run.joined); + assert!(run.end().await.is_err()); assert!(!guard.is_in_flight("branch")); - let retry = guard - .run(WORKSPACE, "branch", Patience::ToTheEnd, |_| async { - Ok(SetupOutcome::SetUp) - }) - .await; - assert_eq!(retry.result, Some(Ok(SetupOutcome::SetUp))); + let retry = guard.start(WORKSPACE, "branch", || async { Ok(SetupOutcome::SetUp) }); + assert_eq!(retry.end().await, Ok(SetupOutcome::SetUp)); } // ── The startup pass ──────────────────────────────────────────────────────── diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs index 50ba7aa2b..c016ba616 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs @@ -316,5 +316,6 @@ async fn setup_refused_over_stranded_commits_fails_with_the_reason() { assert_eq!(rev(clone.path(), "main"), stranded); assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); assert_eq!(host.failure_of(&branch), Some(error)); - assert_eq!(host.retries().len(), 1); + // Left to the error view's Retry: trying again on a timer changes nothing. + assert_eq!(host.retries(), vec![]); } diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/failures.rs b/apps/staged/src-tauri/src/workspace_setup/tests/failures.rs new file mode 100644 index 000000000..b65c315f1 --- /dev/null +++ b/apps/staged/src-tauri/src/workspace_setup/tests/failures.rs @@ -0,0 +1,59 @@ +//! What the reason store announces. The reasons are process-wide, so this +//! is the one test that records any. + +use tauri::Listener; + +use super::*; +use crate::workspace_setup::failures; + +/// A reason is announced when it changes, again only for a setup the user +/// asked for, and dropped once its branch is out of `Error`. +#[test] +fn a_reason_is_announced_once_and_dropped_when_its_branch_moves_on() { + let app = tauri::test::mock_builder() + .build(tauri::test::mock_context(tauri::test::noop_assets())) + .expect("failed to build mock app"); + let (heard_tx, heard) = std::sync::mpsc::channel(); + app.listen(failures::EVENT, move |event| { + let _ = heard_tx.send(event.payload().to_string()); + }); + let heard = || heard.try_iter().collect::>(); + let (store, project) = store_with_project(); + let [failed, moved_on] = ["feature-a", "feature-b"].map(|name| { + let branch = starting_branch(&store, &project, name); + set_status(&store, &branch, WorkspaceStatus::Error) + }); + let set = |branch: &store::Branch, error, user_asked| { + failures::set(app.handle(), &store, &branch.id, error, user_asked) + }; + let event = |branch: &store::Branch, error: &str, user_requested: bool| { + let id = &branch.id; + format!(r#"{{"branchId":"{id}","error":{error},"userRequested":{user_requested}}}"#) + }; + + set(&failed, Some("no network"), false); + set(&moved_on, Some("no network"), false); + let first = [&failed, &moved_on].map(|b| event(b, r#""no network""#, false)); + assert_eq!(heard(), first); + + // An automatic retry failing the same way says nothing; the user's does. + set(&failed, Some("no network"), false); + assert_eq!(heard(), Vec::::new()); + set(&failed, Some("no network"), true); + assert_eq!(heard(), [event(&failed, r#""no network""#, true)]); + + // A poll moved the other branch on: the next failure drops its reason. + set_status(&store, &moved_on, WorkspaceStatus::Running); + set(&failed, Some("no route"), false); + let pruned = [ + event(&failed, r#""no route""#, false), + event(&moved_on, "null", false), + ]; + assert_eq!(heard(), pruned); + assert_eq!(failures::all().get(&moved_on.id), None); + + // Clearing a reason is announced once. + set(&failed, None, true); + set(&failed, None, false); + assert_eq!(heard(), [event(&failed, "null", false)]); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs b/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs index 3e49c16fc..73f476c15 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/requests.rs @@ -1,13 +1,14 @@ -//! What a caller of setup gets back: an answer to what it asked, whatever -//! run it happened to land on, and without being held for a wait that is -//! not its own. +//! What asking for setup does: the `start_workspace` request returns at +//! once and leaves the run to the backend, a caller that waits gets an +//! answer to what it asked, and only a failure of a setup the user asked for +//! is announced as theirs. use std::sync::Arc; use std::time::Duration; -use super::super::guard::{Patience, SetupGuard}; +use super::super::guard::SetupGuard; use super::super::host::{ReadyWait, SetupHost}; -use super::super::{run_guarded, SetupRequest, SetupTrigger}; +use super::super::{request_setup, SetupTrigger}; use super::verify::{running_branch, running_host}; use super::{ eventually, head_branch, origin_and_workstation_clone, set_status, set_up, set_up_under, @@ -15,25 +16,34 @@ use super::{ }; use crate::store::{self, Store, WorkspaceStatus}; -/// A request as the `start_workspace` command makes it: it stays while the -/// run is working and leaves when the run is only waiting. -async fn request( +/// A request as the `start_workspace` command makes it. +fn request( guard: &Arc, store: &Arc, host: &Arc, branch: &store::Branch, -) -> Result, String> { +) -> Result<(), String> { let host: Arc = host.clone(); - let request = SetupRequest::new(SetupTrigger::Requested); - run_guarded( - guard, - store, - &host, - &branch.id, - request, - Patience::WhileWorking, - ) - .await + request_setup(guard, store, &host, &branch.id) +} + +type Waited = tokio::task::JoinHandle, String>>; + +/// A caller that waits on its setup, on a task of its own. +fn spawn_set_up( + guard: &Arc, + store: &Arc, + host: &Arc, + branch: &store::Branch, + trigger: SetupTrigger, +) -> Waited { + let (guard, store, host, branch) = ( + Arc::clone(guard), + Arc::clone(store), + Arc::clone(host), + branch.clone(), + ); + tokio::spawn(async move { set_up_under(&guard, &store, &host, &branch, trigger).await }) } /// A run in a gated checkout, holding its workstation's turn. @@ -42,18 +52,8 @@ async fn hold_the_workstation( store: &Arc, host: &Arc, branch: &store::Branch, -) -> tokio::task::JoinHandle, String>> { - let holder = { - let (guard, store, host, branch) = ( - Arc::clone(guard), - Arc::clone(store), - Arc::clone(host), - branch.clone(), - ); - tokio::spawn(async move { - set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested).await - }) - }; +) -> Waited { + let holder = spawn_set_up(guard, store, host, branch, SetupTrigger::Requested); host.wait_for_checkouts(1).await; holder } @@ -77,17 +77,7 @@ async fn a_request_that_joins_a_run_with_nothing_to_do_still_gets_its_setup() { // Another branch's run has the workstation, so the poll's run for this // branch queues behind it. let holder = hold_the_workstation(&guard, &store, &host, &first).await; - let observed = { - let (guard, store, host, branch) = ( - Arc::clone(&guard), - Arc::clone(&store), - Arc::clone(&host), - branch.clone(), - ); - tokio::spawn(async move { - set_up_under(&guard, &store, &host, &branch, SetupTrigger::Observed).await - }) - }; + let observed = spawn_set_up(&guard, &store, &host, &branch, SetupTrigger::Observed); eventually("the observed run is in flight", || { guard.is_in_flight(&branch.id) }) @@ -114,12 +104,11 @@ async fn a_request_that_joins_a_run_with_nothing_to_do_still_gets_its_setup() { assert_eq!(host.count("check_out"), 2); } -/// A workstation that is still booting can take minutes. The request is -/// answered `Starting` as soon as the run starts waiting, so it does not -/// hold one of the frontend's setup slots; the run goes on and ends in a -/// checked-out `Running` branch. +/// A workstation that is still booting can take minutes. The request does +/// not wait for it, or for anything: it returns with the run under way, and +/// the run ends in a checked-out `Running` branch with nobody waiting. #[tokio::test] -async fn a_request_does_not_wait_out_a_booting_workstation() { +async fn a_request_returns_at_once_and_the_backend_finishes_the_setup() { let (_origin, clone) = origin_and_workstation_clone(); let (store, project) = store_with_project(); let branch = starting_branch(&store, &project, "feature-new"); @@ -136,13 +125,16 @@ async fn a_request_does_not_wait_out_a_booting_workstation() { let host = Arc::new(host); let guard = Arc::new(SetupGuard::default()); - let answered = request(&guard, &store, &host, &branch).await; + request(&guard, &store, &host, &branch).unwrap(); - assert_eq!(answered, Ok(Some(WorkspaceStatus::Starting))); assert!(guard.is_in_flight(&branch.id)); - assert_eq!(host.count("start_workspace"), 1); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); assert_eq!(host.count("check_out"), 0); + // Asking again while it runs starts nothing more. + host.wait_for_calls("start_workspace", 1).await; + request(&guard, &store, &host, &branch).unwrap(); + // The workstation arrives; nobody is waiting on the run. *host.info.lock().unwrap() = [Info::Status("running")].into_iter().collect(); eventually("the run finishes on its own", || { @@ -151,56 +143,81 @@ async fn a_request_does_not_wait_out_a_booting_workstation() { .await; assert_eq!(head_branch(clone.path()), "feature-new"); assert_eq!(host.count("start_workspace"), 1); + assert_eq!(host.count("check_out"), 1); } -/// The same for a run that is behind another branch's on the workstation, -/// which may itself be waiting for it: the request is answered `Starting` -/// at once, whatever the row said (a failed branch's Retry), and the run -/// takes its turn. +/// The error view's Retry. The row says `Starting` and its reason is gone +/// before the request returns, so the card that asked turns at once, also +/// when the run is queued behind another branch's. Here the run it joins is +/// a retry timer's, waiting for its turn: that run is the user's now, and +/// its failure is announced as theirs. #[tokio::test] -async fn a_request_queued_behind_another_branch_is_answered_at_once() { +async fn a_retry_request_turns_the_card_at_once_and_fails_as_the_users() { let (_origin, clone) = origin_and_workstation_clone(); let (store, project) = store_with_project(); let first = starting_branch(&store, &project, "feature-a"); let branch = starting_branch(&store, &project, "feature-b"); let branch = set_status(&store, &branch, WorkspaceStatus::Error); - let (host, gate) = - FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds).gated(); + let mut host = FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds); + host.checkout_error = Some("could not fetch base branch 'main': connection reset"); + let (host, gate) = host.gated(); let host = Arc::new(host); let guard = Arc::new(SetupGuard::default()); let holder = hold_the_workstation(&guard, &store, &host, &first).await; + let retry = spawn_set_up(&guard, &store, &host, &branch, SetupTrigger::Retry); + eventually("the retry is in flight", || guard.is_in_flight(&branch.id)).await; - let answered = request(&guard, &store, &host, &branch).await; + request(&guard, &store, &host, &branch).unwrap(); - assert_eq!(answered, Ok(Some(WorkspaceStatus::Starting))); - assert!(guard.is_in_flight(&branch.id)); - assert_eq!(host.count("check_out"), 1); - // The row agrees with the answer, so a poll of it does too. assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Starting)); + assert!(host.failures().contains(&(branch.id.clone(), None))); + assert_eq!(host.count("check_out"), 1); gate.add_permits(8); - assert_eq!(holder.await.unwrap(), Ok(Some(WorkspaceStatus::Running))); - eventually("the queued run takes its turn", || { - status_of(&store, &branch) == Some(WorkspaceStatus::Running) + holder.await.unwrap().unwrap_err(); + retry.await.unwrap().unwrap_err(); + assert_eq!(host.count("check_out"), 2); + eventually("the queued run fails", || { + status_of(&store, &branch) == Some(WorkspaceStatus::Error) }) .await; - assert_eq!(head_branch(clone.path()), "feature-b"); + eventually("the failure is announced", || { + host.failure_of(&branch).is_some() + }) + .await; + assert_eq!(host.user_asked_failures(), vec![branch.id.clone()]); } -/// A request that can be answered while the run works is: `Running`, with -/// the branch checked out. +/// A new branch's request finds its row `Starting` with nothing running: +/// its failure is the user's too. A request that joins a run something else +/// started for a `Starting` row (the startup pass, a poll) is not what set +/// it going, and that run fails into the card only. #[tokio::test] -async fn a_request_for_a_running_workstation_is_answered_running() { +async fn only_a_request_that_set_the_run_going_makes_its_failure_the_users() { let (_origin, clone) = origin_and_workstation_clone(); let (store, project) = store_with_project(); - let branch = starting_branch(&store, &project, "feature-new"); - let host = Arc::new(running_host(clone.path())); + let new = starting_branch(&store, &project, "feature-new"); + let left = starting_branch(&store, &project, "feature-left-starting"); + let mut host = FakeHost::new(clone.path(), &[Info::Status("running")], Start::Succeeds); + host.checkout_error = Some("could not fetch base branch 'main': connection reset"); + let (host, gate) = host.gated(); + let host = Arc::new(host); let guard = Arc::new(SetupGuard::default()); - let answered = request(&guard, &store, &host, &branch).await; + // The startup pass's run for the branch a previous launch left behind. + let observed = spawn_set_up(&guard, &store, &host, &left, SetupTrigger::Observed); + host.wait_for_checkouts(1).await; + // A window loads and asks for both. + request(&guard, &store, &host, &left).unwrap(); + request(&guard, &store, &host, &new).unwrap(); + gate.add_permits(8); - assert_eq!(answered, Ok(Some(WorkspaceStatus::Running))); - assert_eq!(head_branch(clone.path()), "feature-new"); + observed.await.unwrap().unwrap_err(); + eventually("both runs fail", || { + host.failure_of(&left).is_some() && host.failure_of(&new).is_some() + }) + .await; + assert_eq!(host.user_asked_failures(), vec![new.id.clone()]); } /// A request for a branch that is already `Running` is told so and starts diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs index fec302a50..f1be8e0eb 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs @@ -1,6 +1,7 @@ //! `Error` has a way out: every failure records why and sets a retry, the -//! retry sets the branch up once its workstation is running, and a refused -//! repair does not turn a branch that was `Running` into a failed one. +//! retry sets the branch up once its workstation is running without the +//! card noticing until it has, and a refused repair does not turn a branch +//! that was `Running` into a failed one. use std::sync::Arc; use std::time::Duration; @@ -61,9 +62,20 @@ fn calls(host: &FakeHost) -> Vec<&'static str> { #[test] fn the_wait_doubles_from_a_minute_to_half_an_hour() { let policy = RetryPolicy::DEFAULT; - let waits: Vec = (0..7).map(|n| policy.delay(n).as_secs()).collect(); + let waits: Vec = (0..7).map(|n| policy.delay(n, 0).as_secs()).collect(); assert_eq!(waits, vec![60, 120, 240, 480, 960, 1800, 1800]); - assert_eq!(policy.delay(u32::MAX), Duration::from_secs(1800)); + assert_eq!(policy.delay(u32::MAX, 0), Duration::from_secs(1800)); +} + +/// The timers a restart sets for every failed branch at once do not all +/// fire in the same instant: each waits up to a fifth longer. +#[test] +fn the_wait_is_spread_by_jitter() { + let policy = RetryPolicy::DEFAULT; + let mut waits = (0..200).map(|random| policy.delay(0, random * 997)); + assert!(waits.all(|wait| (MINUTE..=MINUTE * 6 / 5).contains(&wait))); + assert_eq!(policy.delay(0, 12_000), MINUTE + Duration::from_secs(12)); + assert_eq!(policy.delay(0, 12_001), MINUTE); } /// A failed setup leaves the row in `Error` with its reason announced and a @@ -97,6 +109,72 @@ async fn a_failed_setup_says_why_and_sets_a_retry() { assert_eq!(host.retries()[1], (branch.id.clone(), 2 * MINUTE, 2)); } +/// A retry on its own is not something a window should notice until it has +/// worked. While it runs and after it fails the row says `Error` and the +/// reason is never blanked, so the card neither flips to a spinner nor +/// loses its text, and the failure is not announced as the user's. +#[tokio::test] +async fn a_retry_that_fails_leaves_the_card_as_it_was() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let ticket = fail_once(&guard, &store, &failing_host(clone.path()), &branch).await; + let during = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut host = running_host(clone.path()); + host.checkout_error = Some("could not fetch base branch 'main': connection reset"); + host.meanwhile = Some(Box::new({ + let (store, branch, during) = (Arc::clone(&store), branch.clone(), Arc::clone(&during)); + move || during.lock().unwrap().push(status_of(&store, &branch)) + })); + let host = Arc::new(host); + + fire(&guard, &store, &host, &branch, ticket).await; + + assert_eq!(host.count("check_out"), 1); + assert_eq!( + *during.lock().unwrap(), + vec![Some(WorkspaceStatus::Error); 2] + ); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + let reasons = host.failures(); + assert_eq!(reasons.len(), 1, "{reasons:?}"); + assert!(reasons[0] + .1 + .as_deref() + .unwrap() + .contains("connection reset")); + assert_eq!(host.user_asked_failures(), Vec::::new()); + assert_eq!(host.written(), vec![]); + assert_eq!(host.retries(), vec![(branch.id.clone(), 2 * MINUTE, 2)]); +} + +/// A checkout refused over what is in the clone is not retried on a timer: +/// trying again changes nothing until someone has dealt with it, and then +/// the error view's Retry is theirs to press. The timer that was out for +/// the branch no longer counts either. +#[tokio::test] +async fn a_checkout_refused_over_the_clone_sets_no_timer() { + let (_origin, clone) = origin_and_workstation_clone(); + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-new"); + let guard = Arc::new(SetupGuard::default()); + let ticket = fail_once(&guard, &store, &failing_host(clone.path()), &branch).await; + // The clone is on another branch with uncommitted changes. + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + std::fs::write(clone.path().join("file.txt"), "base\nin progress\n").unwrap(); + let host = Arc::new(running_host(clone.path())); + + fire(&guard, &store, &host, &branch, ticket).await; + + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + let reason = host.failure_of(&branch).unwrap(); + assert!(reason.contains("uncommitted changes"), "{reason}"); + assert_eq!(host.retries(), vec![]); + assert!(!guard.retry_is_armed(&branch.id)); + assert_eq!(head_branch(clone.path()), "feature-a"); +} + /// The timer fires with the workstation running: the branch is set up /// without anyone asking, its reason is cleared and its schedule forgotten. /// The workstation is never started for it. @@ -196,6 +274,8 @@ async fn a_timer_for_a_branch_that_moved_on_does_nothing() { assert_eq!(calls(&host), Vec::<&str>::new()); assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Suspended)); assert!(!guard.retry_is_armed(&branch.id)); + // Its reason goes with it, so no later window is handed a stale one. + assert_eq!(host.failures(), vec![(branch.id.clone(), None)]); fail_once(&guard, &store, &failing, &branch).await; assert_eq!(failing.retries().last().unwrap().1, MINUTE); @@ -216,7 +296,7 @@ async fn a_setup_that_works_forgets_the_schedule() { let status = set_up_under(&guard, &store, &host, &branch, SetupTrigger::Requested).await; assert_eq!(status, Ok(Some(WorkspaceStatus::Running))); - assert_eq!(host.failures(), vec![(branch.id.clone(), None)]); + assert_eq!(host.failure_of(&branch), None); assert!(!guard.retry_is_armed(&branch.id)); let before = calls(&host); fire(&guard, &store, &host, &branch, ticket).await; diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index ade9e0781..0824934fa 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -94,10 +94,11 @@ fn the_startup_pass_is_run_and_acts_on_what_it_finds() { pass, "branches_awaiting_setup(store)", ); - assert_calls( + assert_before( "resume_pending_setups", pass, - "Some(StartupAction::SetUp) => trigger_branch_setup(app_handle, store, &branch.id)", + "Some(StartupAction::SetUp) => {", + "trigger_branch_setup(app_handle, store, &branch.id, false)", ); assert_calls( "resume_pending_setups", @@ -111,10 +112,16 @@ fn the_startup_pass_is_run_and_acts_on_what_it_finds() { #[test] fn a_trigger_runs_setup_and_a_retry_is_armed_on_the_shared_guard() { let trigger = body_of(SETUP, "pub(crate) fn trigger_branch_setup<"); + assert_calls("trigger_branch_setup", trigger, "SetupGuard::global()"); assert_calls( "trigger_branch_setup", trigger, - "ensure_branch_set_up(&app_handle, &store, &branch_id, SetupTrigger::Observed)", + "SetupRequest::new(SetupTrigger::Observed)", + ); + assert_calls( + "trigger_branch_setup", + trigger, + "run_guarded(guard, &store, &host, &branch_id, request)", ); let ensure = body_of(SETUP, "pub(crate) async fn ensure_branch_set_up<"); assert_calls("ensure_branch_set_up", ensure, "run_guarded("); @@ -128,7 +135,7 @@ fn a_trigger_runs_setup_and_a_retry_is_armed_on_the_shared_guard() { assert_calls( "branch_failed", failed, - "failures::set(app_handle, branch_id, Some(reason))", + "failures::set(app_handle, store, branch_id, Some(reason), user_asked)", ); assert_calls( "branch_failed", @@ -143,12 +150,12 @@ fn a_trigger_runs_setup_and_a_retry_is_armed_on_the_shared_guard() { fn the_real_host_announces_retries_and_reconciles() { let reason = body_of( HOST, - " fn failure_changed(&self, branch_id: &str, error: Option<&str>) {\n failures", + " fn failure_changed(&self, branch_id: &str, error: Option<&str>, user_asked: bool) {\n failures", ); assert_calls( "BloxHost::failure_changed", reason, - "failures::set(&self.app_handle, branch_id, error)", + "failures::set(&self.app_handle, &self.store, branch_id, error, user_asked)", ); let later = body_of( @@ -181,7 +188,7 @@ fn the_pollers_effects_reach_setup() { let effects = body_of(POLL, "fn app_effects<"); for call in [ "branches::workspace_status_written(app_handle, branch_id, changed, status)", - "super::trigger_branch_setup(app_handle, store, branch_id)", + "super::trigger_branch_setup(app_handle, store, branch_id, false)", "super::arm_retry(app_handle, store, branch_id)", "super::branch_failed(app_handle, store, branch_id, reason)", ] { @@ -203,14 +210,15 @@ fn the_pollers_effects_reach_setup() { } /// A resume that worked leaves its branches `Starting` and triggers setup -/// for each; one that failed gives each its reason and its retry. +/// for each, as the user's ask; one that failed gives each its reason and +/// its retry. #[test] fn a_resume_triggers_setup_or_says_why_it_failed() { let resume = body_of(BRANCHES, "pub(crate) async fn resume_workspace_impl("); assert_calls( "resume_workspace_impl", resume, - "crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id)", + "crate::workspace_setup::trigger_branch_setup(app_handle, store, branch_id, true)", ); assert_calls( "resume_workspace_impl", @@ -223,35 +231,35 @@ fn a_resume_triggers_setup_or_says_why_it_failed() { ); } -/// The command and its web arm ask for setup the same way and return the -/// row's resulting status rather than nothing. +/// The command and its web arm only ask for setup: neither waits for the +/// run, whose outcome reaches a client through the row and the reason event. #[test] -fn start_workspace_returns_what_setup_left_the_row_in() { +fn start_workspace_only_asks_for_setup() { let command = body_of(BRANCHES, "pub async fn start_workspace("); + assert_calls("start_workspace", command, "Result<(), String>"); assert_calls( "start_workspace", command, - "Result", - ); - assert_calls( - "start_workspace", - command, - "crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id).await", + "crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id)", ); + let arm = web_arm("start_workspace"); assert_calls( "the start_workspace web arm", - web_arm("start_workspace"), - "crate::workspace_setup::request_branch_setup(&app_handle, &store, &branch_id)", + arm, + "crate::workspace_setup::request_branch_setup(app_handle, &store, &branch_id)", ); + for (what, body) in [("start_workspace", command), ("its web arm", arm)] { + assert!(!body.contains(".await"), "{what} must not wait on setup"); + } - let request = body_of(SETUP, "pub(crate) async fn request_branch_setup<"); + let request = body_of(SETUP, "pub(crate) fn request_setup("); assert_calls( - "request_branch_setup", + "request_setup", request, "SetupRequest::new(SetupTrigger::Requested)", ); - // Not held for a workstation's boot. - assert_calls("request_branch_setup", request, "Patience::WhileWorking"); + assert_calls("request_setup", request, "start_run("); + assert!(!request.contains(".await"), "a request must not wait"); assert_calls( "the invoke handler", diff --git a/apps/staged/src-tauri/src/workspace_setup/verify.rs b/apps/staged/src-tauri/src/workspace_setup/verify.rs index b37aad276..fd12ff2cd 100644 --- a/apps/staged/src-tauri/src/workspace_setup/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/verify.rs @@ -46,7 +46,7 @@ use crate::branches; use crate::store::{self, Store}; use super::clone::checkout_for; -use super::guard::{Patience, SetupGuard}; +use super::guard::SetupGuard; use super::host::{blox_host, Checkout, SetupHost}; use super::{run_guarded, SetupRequest}; @@ -278,21 +278,15 @@ pub(crate) async fn verify_session_checkout( session starts" ); - run_guarded( - guard, - store, - host, - &branch.id, - SetupRequest::repair_for(session_id), - Patience::ToTheEnd, - ) - .await - .map_err(|e| { - format!( - "Workspace '{workspace}' had {found} checked out, where this session's branch \ + let repair = SetupRequest::repair_for(session_id); + run_guarded(guard, store, host, &branch.id, repair) + .await + .map_err(|e| { + format!( + "Workspace '{workspace}' had {found} checked out, where this session's branch \ '{expected}' was expected, and it could not be switched: {e}" - ) - })?; + ) + })?; // The repair's own word is not taken for it: a run this one joined, or a // row that had left `Running` and so had nothing to repair, says nothing diff --git a/apps/staged/src-tauri/src/workspace_setup/workstation.rs b/apps/staged/src-tauri/src/workspace_setup/workstation.rs index 686747812..4d6dc207a 100644 --- a/apps/staged/src-tauri/src/workspace_setup/workstation.rs +++ b/apps/staged/src-tauri/src/workspace_setup/workstation.rs @@ -7,7 +7,6 @@ use crate::blox::BloxError; use crate::branches; use crate::store::{self, Store, WorkspaceStatus}; -use super::guard::Progress; use super::host::{reports_running, Checkout, SetupHost}; /// `sq blox ws start`, and when that fails without ruling the workstation @@ -17,7 +16,6 @@ pub(super) async fn start_workstation( host: &dyn SetupHost, branch: &store::Branch, checkout: &Checkout, - progress: &Progress, ) -> Result<(), String> { let workspace = checkout.workspace.as_str(); let source = format!( @@ -55,7 +53,6 @@ pub(super) async fn start_workstation( branch, workspace, &format!("`sq blox ws start` failed: {start_error}"), - progress, ) .await } @@ -63,29 +60,12 @@ pub(super) async fn start_workstation( /// Wait, within [`SetupHost::ready_wait`], for the workstation to report /// running. `why` says what the wait was for, for the error when it never /// does. -/// -/// The run says it is waiting for as long as this takes, so a caller that -/// only stays while the run is working ([`Patience::WhileWorking`]) leaves. pub(super) async fn wait_until_running( store: &Arc, host: &dyn SetupHost, branch: &store::Branch, workspace: &str, why: &str, - progress: &Progress, -) -> Result<(), String> { - progress.waiting_for_workstation(); - let result = poll_until_running(store, host, branch, workspace, why).await; - progress.working(); - result -} - -async fn poll_until_running( - store: &Arc, - host: &dyn SetupHost, - branch: &store::Branch, - workspace: &str, - why: &str, ) -> Result<(), String> { let wait = host.ready_wait(); let started = tokio::time::Instant::now(); diff --git a/apps/staged/src/lib/commands.ts b/apps/staged/src/lib/commands.ts index 7a759c92a..2ae2b6382 100644 --- a/apps/staged/src/lib/commands.ts +++ b/apps/staged/src/lib/commands.ts @@ -786,14 +786,13 @@ export function createRemoteBranch( } /** - * Set up a remote branch on its Blox workstation (or retry a branch in - * `error`). Resolves with the status the branch row holds afterwards: - * `running` once it is checked out and verified, `starting` when the backend - * returned early while the workstation boots (setup carries on there and its - * outcome lands on the row), or whatever else the row holds now. Rejects with - * the reason when setup failed and left the branch in `error`. - */ -export function startWorkspace(branchId: string): Promise { + * Ask the backend to set up a remote branch on its Blox workstation (or to + * retry one in `error`). Only asks: it resolves once the backend's setup run + * is under way, the branch's status says how it ends, and a failure's reason + * arrives as a `workspace-setup-error` event. Rejects only when the request + * itself could not be made. + */ +export function startWorkspace(branchId: string): Promise { return invokeCommand('start_workspace', { branchId }); } diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index ff5283183..e13489053 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -12,17 +12,14 @@ interface LocalSetupTask { run: () => Promise; } -interface RemoteSetupTask { - branchId: string; - workspaceName: string | null; - run: () => Promise; -} - /** Payload of the backend's `workspace-setup-error` event: why a setup run - * left the branch in `error`, or null when a new run for it begins. */ + * left the branch in `error`, or null once it has no reason any more. + * `userRequested` marks the failure of a setup the user asked for (a new + * branch, Retry, Resume), as opposed to one the backend ran on its own. */ interface WorkspaceSetupErrorEvent { branchId: string; error: string | null; + userRequested?: boolean; } interface WorkspaceLifecycleHooks { @@ -48,11 +45,6 @@ class WorkspaceLifecycleController { private pendingSetupBranches = new Set(); private queuedSetupBranches = new Set(); - /** Remote branches whose `startWorkspace` answered `starting`: the backend - * returned early while their workstation boots and carries on setting - * them up itself, so nothing here re-asks, and the outcome arrives via - * the status poll (and the reason, on failure, via the error event). */ - private backendSetupBranches = new Set(); /** Projects a transfer just ended for, skipped until their refetch lands: * the lists held until then describe the old location. */ private awaitingTransferRefresh = new Set(); @@ -61,30 +53,18 @@ class WorkspaceLifecycleController { private prerunOnSetup = new Set(); private activeLocalSetupCount = 0; - private activeRemoteSetupCount = 0; private readonly MAX_LOCAL_SETUP_CONCURRENCY = 1; - private readonly MAX_REMOTE_SETUP_CONCURRENCY = 4; - private activeRemoteWorkspaceStarts = new Set(); private localSetupTaskQueue: LocalSetupTask[] = []; - private remoteSetupTaskQueue: RemoteSetupTask[] = []; // Single adaptive poller: 3s while any workspace is starting, 30s while - // any is running, stopped otherwise. + // any is running, stopped otherwise. A remote branch's setup is the + // backend's from start to finish: this controller asks for it, and the + // poll and the `workspace-setup-error` event say how it went. It keeps no + // clock of its own on a `starting` branch; the backend's waits are bounded + // and end in `running` or `error` on the row. private readonly STARTING_POLL_MS = 3000; private readonly RUNNING_POLL_MS = 30_000; - // How long a branch may sit in `starting` with nothing setting it up - // before it is shown as failed. A branch stays `starting` until the - // backend has checked it out on the workstation, not just until the - // workstation is up. That setup is bounded and lands its own outcome on - // the row (`running`, or `error` with a reason sent as a - // `workspace-setup-error` event, and as a rejection of a `startWorkspace` - // call joined to it). So the clock only runs for a branch with no setup - // call queued or in flight here, and whose last `startWorkspace` did not - // answer `starting` (the backend returning early while the workstation - // boots, and carrying on with the setup itself). - private readonly STARTING_TIMEOUT_MS = 5 * 60 * 1000; - /** Bumped by start() and stop(), so an answer arriving after a restart * does not write into the next run's state. */ private generation = 0; @@ -96,17 +76,16 @@ class WorkspaceLifecycleController { private pollTimer: ReturnType | null = null; private pollInFlight = false; - private startingTimestamps = new Map(); private kickoffTimer: ReturnType | null = null; start(hooks: WorkspaceLifecycleHooks): void { this.hooks = hooks; const generation = ++this.generation; this.unlistenSetupErrors?.(); - // The backend reports every failed setup run, including the ones no call - // from here was joined to (its startup pass, poll-triggered runs and - // automatic retries). The snapshot covers failures from before this - // listener went live, and is retaken on each web-socket reconnect. + // The backend reports every failed setup run, whoever asked for it (its + // startup pass, poll-triggered runs and automatic retries included). The + // snapshot covers failures from before this listener went live, and is + // retaken on each web-socket reconnect. this.unlistenSetupErrors = listenToEvent( 'workspace-setup-error', (payload) => { @@ -130,15 +109,10 @@ class WorkspaceLifecycleController { } this.pendingSetupBranches.clear(); this.queuedSetupBranches.clear(); - this.backendSetupBranches.clear(); this.awaitingTransferRefresh.clear(); this.prerunOnSetup.clear(); - this.activeRemoteWorkspaceStarts.clear(); - this.startingTimestamps.clear(); this.localSetupTaskQueue = []; - this.remoteSetupTaskQueue = []; this.activeLocalSetupCount = 0; - this.activeRemoteSetupCount = 0; this.hooks = null; } @@ -221,19 +195,6 @@ class WorkspaceLifecycleController { this.clearWorkspaceError(branchId); } - if (workspaceStatus !== 'starting') { - this.backendSetupBranches.delete(branchId); - } - - // Track starting timestamps for timeout logic. - if (workspaceStatus === 'starting') { - if (!this.startingTimestamps.has(branchId)) { - this.startingTimestamps.set(branchId, Date.now()); - } - } else { - this.startingTimestamps.delete(branchId); - } - // Kick the poller when a workspace enters an active state. Queued // sessions are not drained from here: the backend kicks the branch queue // itself when it writes the `running` status. @@ -248,13 +209,13 @@ class WorkspaceLifecycleController { /** * Set a remote branch up again, typically one in `error`: the backend puts - * it back to `starting` and reruns setup. A no-op while a setup call for - * the branch is already queued or in flight here. + * it back to `starting` and reruns setup. A no-op while a request for the + * branch is already out. */ retryWorkspace(projectId: string, branchId: string): void { const hooks = this.hooks; if (!hooks || this.isProjectHeld(projectId)) return; - if (this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId)) return; + if (this.pendingSetupBranches.has(branchId)) return; const branch = hooks .getBranchesByProject() .get(projectId) @@ -265,10 +226,9 @@ class WorkspaceLifecycleController { const location = hooks.getProjectLocation(projectId); if (location && location !== 'remote') return; - this.backendSetupBranches.delete(branchId); this.clearWorkspaceError(branchId); this.handleWorkspaceStatusChange(projectId, branchId, 'starting'); - this.queueRemoteSetup(projectId, branchId, branch.workspaceName); + this.requestRemoteSetup(branchId); } async resumeWorkspace(projectId: string, workspaceName: string): Promise { @@ -335,16 +295,11 @@ class WorkspaceLifecycleController { clearBranchState(branchId: string): void { this.pendingSetupBranches.delete(branchId); this.queuedSetupBranches.delete(branchId); - this.backendSetupBranches.delete(branchId); - this.startingTimestamps.delete(branchId); this.prerunOnSetup.delete(branchId); this.localSetupTaskQueue = this.localSetupTaskQueue.filter( (task) => task.branchId !== branchId ); - this.remoteSetupTaskQueue = this.remoteSetupTaskQueue.filter( - (task) => task.branchId !== branchId - ); let changed = false; if (this.worktreeErrors.has(branchId)) { @@ -373,16 +328,6 @@ class WorkspaceLifecycleController { ); } - /** Whether a setup call for the branch is queued or in flight here, or - * the backend said it carries on with the branch's setup itself. */ - private isSetupOwned(branchId: string): boolean { - return ( - this.pendingSetupBranches.has(branchId) || - this.queuedSetupBranches.has(branchId) || - this.backendSetupBranches.has(branchId) - ); - } - private setWorkspaceError(branchId: string, message: string): void { if (this.workspaceErrors.get(branchId) === message) return; this.workspaceErrors = new Map(this.workspaceErrors).set(branchId, message); @@ -397,14 +342,19 @@ class WorkspaceLifecycleController { this.version++; } - /** Not toasted: a failure a call from here was joined to is toasted when - * that call rejects. */ - private applySetupErrorEvent({ branchId, error }: WorkspaceSetupErrorEvent): void { + /** Every failure lands on the card. Only one the backend marks as the + * user's own ask is toasted: its automatic retries, startup pass and + * poll-triggered runs fail without interrupting anyone. */ + private applySetupErrorEvent(event: WorkspaceSetupErrorEvent): void { + const { branchId, error } = event; this.setupErrorEventSeqByBranch.set(branchId, ++this.setupErrorEventSeq); if (error == null) { this.clearWorkspaceError(branchId); - } else { - this.setWorkspaceError(branchId, error); + return; + } + this.setWorkspaceError(branchId, error); + if (event.userRequested) { + toast.error('Unable to start workspace', { description: error, duration: Infinity }); } } @@ -474,28 +424,7 @@ class WorkspaceLifecycleController { return; } - // Apply 5-minute timeout to starting workspaces before polling. Not to - // one whose setup call is still out: its answer settles the branch, and - // timing out under it would show an error over a setup that then - // succeeds. - const now = Date.now(); - const activeBranchIds: string[] = []; - for (const target of targets) { - if (target.status === 'starting' && !this.isSetupOwned(target.branchId)) { - const startedAt = this.startingTimestamps.get(target.branchId); - if (startedAt && now - startedAt > this.STARTING_TIMEOUT_MS) { - this.startingTimestamps.delete(target.branchId); - this.handleWorkspaceStatusChange(target.projectId, target.branchId, 'error'); - continue; - } - } - activeBranchIds.push(target.branchId); - } - - if (activeBranchIds.length === 0) { - this.scheduleNextPoll(); - return; - } + const activeBranchIds = targets.map((target) => target.branchId); this.pollInFlight = true; try { @@ -570,12 +499,7 @@ class WorkspaceLifecycleController { private enqueueBranchSetup(projectId: string, branch: Branch): void { const branchId = branch.id; - // The backend's own setup run is over once the row leaves `starting`, - // whichever of the poll or a refetched list reports it first. - if (branch.branchType === 'remote' && branch.workspaceStatus !== 'starting') { - this.backendSetupBranches.delete(branchId); - } - if (this.isSetupOwned(branchId)) return; + if (this.pendingSetupBranches.has(branchId) || this.queuedSetupBranches.has(branchId)) return; // A project's branches all live where it does. One that doesn't is from // a list read before a transfer switched the project over, and setting @@ -598,67 +522,33 @@ class WorkspaceLifecycleController { } if (branch.branchType === 'remote' && branch.workspaceStatus === 'starting') { - this.queueRemoteSetup(projectId, branchId, branch.workspaceName); + this.requestRemoteSetup(branchId); } } - private queueRemoteSetup( - projectId: string, - branchId: string, - workspaceName: string | null - ): void { - this.queuedSetupBranches.add(branchId); - this.remoteSetupTaskQueue.push({ - branchId, - workspaceName, - run: () => this.runRemoteSetup(projectId, branchId), - }); - // A branch can turn `starting` in a refetched list (the backend retrying - // it on its own) with the poller stopped; its outcome comes from polls. - this.ensurePolling(); - this.pumpRemoteSetupQueue(); - } - - private async runRemoteSetup(projectId: string, branchId: string): Promise { - const generation = this.generation; + /** + * Ask the backend to set a remote branch up. Only asks: the call resolves + * as soon as the backend's run is under way (or joined), the status poll + * reports how it ends, and a failure's reason arrives as a + * `workspace-setup-error` event. A rejection means the request itself + * could not be made, and says nothing about the branch. Asking again for a + * branch that is still `starting` is harmless: it joins the same run. + */ + private requestRemoteSetup(branchId: string): void { this.pendingSetupBranches.add(branchId); - this.clearWorkspaceError(branchId); - - try { - // Joins the backend's setup run for the branch (starting one if none - // is running). Resolves with the row's status: `running` once the - // branch is checked out and verified, `starting` when the backend - // returned early while the workstation boots, or whatever else the - // row holds now. Rejects with why it is in `error`. - const result = await commands.startWorkspace(branchId); - if (this.generation !== generation) return; - const status = this.toWorkspaceStatus(result?.status ?? ''); - if (!status) { - // Nothing to apply: leave the branch to the poll, without asking - // again on every kickoff meanwhile. - console.warn('[workspaceLifecycle] Unknown startWorkspace status:', result); - this.backendSetupBranches.add(branchId); - this.ensurePolling(); - return; - } - if (status === 'starting') { - // Setup carries on in the backend: the poll reports how it ends. - this.backendSetupBranches.add(branchId); - } - this.handleWorkspaceStatusChange(projectId, branchId, status, result.workstationId); - } catch (e) { - if (this.generation !== generation) return; - console.error('[workspaceLifecycle] Failed to start workspace:', e); - const message = this.errorMessage(e); - this.setWorkspaceError(branchId, message); - this.handleWorkspaceStatusChange(projectId, branchId, 'error'); - toast.error('Unable to start workspace', { - description: message, - duration: Infinity, + // A branch can turn `starting` in a refetched list with the poller + // stopped; its outcome comes from polls. + this.ensurePolling(); + commands + .startWorkspace(branchId) + .catch((e) => { + console.error('[workspaceLifecycle] Failed to request workspace setup:', e); + // A reason a Retry cleared here may still stand in the backend. + void this.seedSetupErrors(this.generation); + }) + .finally(() => { + this.pendingSetupBranches.delete(branchId); }); - } finally { - if (this.generation === generation) this.pendingSetupBranches.delete(branchId); - } } private pumpLocalSetupQueue(): void { @@ -683,37 +573,6 @@ class WorkspaceLifecycleController { } } - private pumpRemoteSetupQueue(): void { - while ( - this.activeRemoteSetupCount < this.MAX_REMOTE_SETUP_CONCURRENCY && - this.remoteSetupTaskQueue.length > 0 - ) { - const taskIndex = this.remoteSetupTaskQueue.findIndex( - (task) => !task.workspaceName || !this.activeRemoteWorkspaceStarts.has(task.workspaceName) - ); - if (taskIndex === -1) break; - const task = this.remoteSetupTaskQueue.splice(taskIndex, 1)[0]; - - this.activeRemoteSetupCount += 1; - if (task.workspaceName) { - this.activeRemoteWorkspaceStarts.add(task.workspaceName); - } - task - .run() - .catch((e) => { - console.error('[workspaceLifecycle] Remote workspace setup task failed:', e); - }) - .finally(() => { - this.activeRemoteSetupCount = Math.max(0, this.activeRemoteSetupCount - 1); - if (task.workspaceName) { - this.activeRemoteWorkspaceStarts.delete(task.workspaceName); - } - this.queuedSetupBranches.delete(task.branchId); - this.pumpRemoteSetupQueue(); - }); - } - } - private async setupBranchWorktree( branchId: string, projectId: string, diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 89a004a2d..0d3a5deab 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -11,13 +11,13 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { let projectLocations: Map; let refreshTransferredProject: Mock<(projectId: string) => Promise>; let eventListeners: Map void>>; + let toastError: ReturnType; /** Deliver a backend event to the controller's listeners. */ function emit(event: string, payload: unknown) { for (const listener of eventListeners.get(event) ?? []) listener(payload); } - const RUNNING = { status: 'running', workstationId: null }; const STARTING = { status: 'starting', workstationId: null }; beforeEach(() => { @@ -34,7 +34,7 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { commandsMock = { setupWorktree: vi.fn(), setupWorktreeAndRunPrerun, - startWorkspace: vi.fn(async () => RUNNING), + startWorkspace: vi.fn(async () => undefined), pollAllWorkspaceStatuses: vi.fn(async () => ({})), getWorkspaceSetupErrors: vi.fn(async () => ({})), }; @@ -58,7 +58,10 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { }; }, })); - vi.doMock('svelte-sonner', () => ({ toast: { error: vi.fn() } })); + vi.doMock('svelte-sonner', () => { + toastError = vi.fn(); + return { toast: { error: toastError } }; + }); vi.doMock('../settings/preferences.svelte', () => ({ getPreferredAgent: () => 'claude', })); @@ -281,78 +284,24 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); - // A branch stays `starting` until the backend has checked it out on the - // workstation, which can outlast the five-minute starting timeout. While - // the setup call is out, its answer settles the branch: timing out under - // it would show an error over a setup that then succeeds. - it('does not time out a starting branch while its setup call is in flight', async () => { + // The call only asks. A rejection means the request could not be made and + // says nothing about the branch: its status is the poll's to report and + // its reason the backend's, so the reasons are read again. + it('does not read a rejected startWorkspace as a failed setup', async () => { vi.useFakeTimers(); - let finishSetup!: (result: typeof RUNNING) => void; - commandsMock.startWorkspace.mockReturnValue( - new Promise((resolve) => (finishSetup = resolve)) - ); - // The poll never promotes it: only setup does. - commandsMock.pollAllWorkspaceStatuses.mockResolvedValue({ - 'branch-1': { status: 'starting', workstationId: null }, - }); + commandsMock.startWorkspace.mockRejectedValue(new Error('Branch has no workspace name')); branchesByProject = new Map([ ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], ]); const lifecycle = await newController(); - await kickoff(lifecycle); - expect(commandsMock.startWorkspace).toHaveBeenCalledWith('branch-1'); - - await vi.advanceTimersByTimeAsync(7 * 60 * 1000); - expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('starting'); - expect(lifecycle.getWorkspaceErrors().size).toBe(0); - - finishSetup(RUNNING); await settle(); - expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('running'); - lifecycle.stop(); - }); - - // The timeout still catches a branch stuck in `starting` that nothing here - // is setting up, so it cannot spin forever. - it('times out a starting branch it has no setup call for', async () => { - vi.useFakeTimers(); - commandsMock.pollAllWorkspaceStatuses.mockResolvedValue({ - 'branch-1': { status: 'starting', workstationId: null }, - }); - branchesByProject = new Map([ - ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], - ]); - // Setup skips a branch whose type disagrees with its project's location. - projectLocations = new Map([['project-1', 'local']]); - const lifecycle = await newController(); + commandsMock.getWorkspaceSetupErrors.mockResolvedValue({ 'branch-1': 'could not fetch' }); await kickoff(lifecycle); - expect(commandsMock.startWorkspace).not.toHaveBeenCalled(); - await vi.advanceTimersByTimeAsync(4 * 60 * 1000); expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('starting'); - await vi.advanceTimersByTimeAsync(2 * 60 * 1000); - expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('error'); - lifecycle.stop(); - }); - - // A setup that fails leaves the branch in `error` in the backend and - // rejects; the branch shows the reason and is no longer polled. - it('shows the setup error when startWorkspace rejects', async () => { - vi.useFakeTimers(); - commandsMock.startWorkspace.mockRejectedValue( - new Error("Failed to set up 'acme/widgets' in workspace 'ws': could not fetch") - ); - branchesByProject = new Map([ - ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'starting' }]], - ]); - const lifecycle = await newController(); - await kickoff(lifecycle); - - expect(branchesByProject.get('project-1')?.[0].workspaceStatus).toBe('error'); - expect(lifecycle.getWorkspaceErrors().get('branch-1')).toContain('could not fetch'); - const polls = commandsMock.pollAllWorkspaceStatuses.mock.calls.length; - await vi.advanceTimersByTimeAsync(60 * 1000); - expect(commandsMock.pollAllWorkspaceStatuses.mock.calls.length).toBe(polls); + expect(commandsMock.getWorkspaceSetupErrors).toHaveBeenCalledTimes(2); + expect(lifecycle.getWorkspaceErrors().get('branch-1')).toBe('could not fetch'); + expect(toastError).not.toHaveBeenCalled(); lifecycle.stop(); }); @@ -387,35 +336,15 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { return branchesByProject.get('project-1')?.find((b) => b.id === branchId)?.workspaceStatus; } - // The resolved status is the row's, not an inference: a setup call can - // come back with the branch suspended (or anything else) as well. - it('applies the status startWorkspace resolves with', async () => { - vi.useFakeTimers(); - commandsMock.startWorkspace.mockImplementation(async (branchId: string) => - branchId === 'branch-1' - ? { status: 'running', workstationId: 7 } - : { status: 'suspended', workstationId: null } - ); - branchesByProject = new Map([['project-1', startingBranches('branch-1', 'branch-2')]]); - const lifecycle = await newController(); - await kickoff(lifecycle); - - expect(branchesByProject.get('project-1')).toEqual([ - { id: 'branch-1', branchType: 'remote', workspaceStatus: 'running', workstationId: 7 }, - { id: 'branch-2', branchType: 'remote', workspaceStatus: 'suspended' }, - ]); - lifecycle.stop(); - }); - - // `starting` means the backend returned early while the workstation boots - // and carries on with the setup itself. The call must not keep a setup - // slot, be re-issued on the next kickoff, or let the timeout fire under - // the backend's run; the poll reports how it ends. - it('releases the slot on a starting answer and leaves the outcome to the poll', async () => { + // The backend runs the setup and bounds it. Every `starting` branch is + // asked for at once and once, nothing here times a slow one out, and the + // poll reports how each ends. + it('asks for every starting branch once and leaves the outcome to the poll', async () => { vi.useFakeTimers(); const ids = ['branch-1', 'branch-2', 'branch-3', 'branch-4', 'branch-5']; + // One request never returns: it holds up no other and is not re-issued. commandsMock.startWorkspace.mockImplementation((branchId: string) => - branchId === 'branch-5' ? new Promise(() => {}) : Promise.resolve(STARTING) + branchId === 'branch-5' ? new Promise(() => {}) : Promise.resolve() ); let polled: Record = Object.fromEntries(ids.map((id) => [id, STARTING])); @@ -426,23 +355,34 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { expect(commandsMock.startWorkspace.mock.calls.map(([id]) => id)).toEqual(ids); await kickoff(lifecycle); - expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(5); + expect(commandsMock.startWorkspace.mock.calls.map(([id]) => id)).toEqual([ + ...ids, + ...ids.slice(0, 4), + ]); await vi.advanceTimersByTimeAsync(7 * 60 * 1000); expect(ids.map(statusOf)).toEqual(ids.map(() => 'starting')); expect(lifecycle.getWorkspaceErrors().size).toBe(0); - polled = { ...polled, 'branch-1': { status: 'running', workstationId: 3 } }; + polled = { + ...polled, + 'branch-1': { status: 'running', workstationId: 3 }, + 'branch-2': { status: 'error', workstationId: null }, + }; await vi.advanceTimersByTimeAsync(3000); - expect(statusOf('branch-1')).toBe('running'); + expect(branchesByProject.get('project-1')?.slice(0, 2)).toEqual([ + { id: 'branch-1', branchType: 'remote', workspaceStatus: 'running', workstationId: 3 }, + { id: 'branch-2', branchType: 'remote', workspaceStatus: 'error' }, + ]); + expect(toastError).not.toHaveBeenCalled(); lifecycle.stop(); }); - // A list refetched after the backend retried a branch on its own (error to - // starting) joins that run and polls it, even with the poller stopped. - it('joins and polls a branch a refetched list shows starting again', async () => { + // A list refetched with a branch back in `starting` (a Retry or a resume + // from another window) asks for it and polls it, even with the poller + // stopped. + it('asks for and polls a branch a refetched list shows starting again', async () => { vi.useFakeTimers(); - commandsMock.startWorkspace.mockResolvedValue(STARTING); branchesByProject = new Map([ ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'error' }]], ]); @@ -458,37 +398,67 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); - it('retryWorkspace reruns setup for a branch in error, once at a time', async () => { + it('retryWorkspace asks again for a branch in error, once at a time', async () => { vi.useFakeTimers(); - commandsMock.startWorkspace.mockRejectedValueOnce(new Error('could not fetch')); - branchesByProject = new Map([['project-1', startingBranches('branch-1')]]); + branchesByProject = new Map([ + ['project-1', [{ id: 'branch-1', branchType: 'remote', workspaceStatus: 'error' }]], + ]); const lifecycle = await newController(); - await kickoff(lifecycle); - expect(statusOf('branch-1')).toBe('error'); + await settle(); + emit('workspace-setup-error', { branchId: 'branch-1', error: 'could not fetch' }); expect(lifecycle.getWorkspaceErrors().get('branch-1')).toBe('could not fetch'); - let finishSetup!: (result: typeof RUNNING) => void; + let finishRequest!: () => void; commandsMock.startWorkspace.mockReturnValueOnce( - new Promise((resolve) => (finishSetup = resolve)) + new Promise((resolve) => (finishRequest = resolve)) ); lifecycle.retryWorkspace('project-1', 'branch-1'); - await settle(); expect(statusOf('branch-1')).toBe('starting'); expect(lifecycle.getWorkspaceErrors().has('branch-1')).toBe(false); - expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(2); + expect(commandsMock.pollAllWorkspaceStatuses).toHaveBeenCalledWith(['branch-1']); lifecycle.retryWorkspace('project-1', 'branch-1'); + expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(1); + + finishRequest(); await settle(); + lifecycle.retryWorkspace('project-1', 'branch-1'); expect(commandsMock.startWorkspace).toHaveBeenCalledTimes(2); + lifecycle.stop(); + }); - finishSetup(RUNNING); + // The backend says whose run failed. Its own (an automatic retry, the + // startup pass, a poll-triggered run) only updates the card; one the user + // asked for (a new branch, Retry, Resume) also raises a toast. + it('shows an automatic retry failing on the card without a toast', async () => { + const lifecycle = await newController(); await settle(); - expect(statusOf('branch-1')).toBe('running'); + + emit('workspace-setup-error', { + branchId: 'branch-1', + error: 'could not fetch', + userRequested: false, + }); + emit('workspace-setup-error', { branchId: 'branch-2', error: 'workstation never booted' }); + expect([...lifecycle.getWorkspaceErrors()]).toEqual([ + ['branch-1', 'could not fetch'], + ['branch-2', 'workstation never booted'], + ]); + expect(toastError).not.toHaveBeenCalled(); + + emit('workspace-setup-error', { + branchId: 'branch-1', + error: 'still could not fetch', + userRequested: true, + }); + expect(lifecycle.getWorkspaceErrors().get('branch-1')).toBe('still could not fetch'); + expect(toastError.mock.calls).toEqual([ + ['Unable to start workspace', { description: 'still could not fetch', duration: Infinity }], + ]); lifecycle.stop(); }); - // The backend reports failures of runs nothing here joined (its startup - // pass, poll triggers, automatic retries), and a snapshot covers the ones + // The backend reports every failed run, and a snapshot covers the ones // from before the window mounted. it('keeps setup reasons from the error event and the snapshot taken at start', async () => { let finishSnapshot!: (errors: Record) => void; From c4e9628881cdad1c1393575e5175f005fd1eb7d6 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Wed, 18 Nov 2026 15:00:50 +1100 Subject: [PATCH 5/9] fix(staged): hold the queue behind a refused repair and settle three more review warnings Review f28fe49a left four warnings open after 67dffa06. This closes them with the least code each needs, and adds the tests the review listed as missing for them. A refused repair no longer ends a branch's queued sessions one after another. A clone a sibling branch's session works in is not repaired at all: the branch stays Running, the asking session fails with a message that names the sibling's session, and the drain holds the row. A held row spends no queue attempt, is asked about again from the database every five seconds, and is released when the sibling's session is done. A start that leaves the branch out of Running spends no attempt either. A repair refused over what is in the clone (changes in the way, commits that would be left behind) now leaves the branch in Error with its reason and no timer, where the queue already holds and Retry releases it, in place of the Running that 53fb152a wrote back. Commits stranded on the local base are adopted only when the owner can be told. Setup refuses when another branch row sharing the clone has no local branch either. It also refuses when not every stranded commit was made by `git commit` in this clone, read from HEAD's reflog: ancestry cannot tell a force-pushed base from one that moved on, and the reflog shows whether the commits were written here or fetched. When commits are adopted the user is told through the workspace-setup-error event, which gains an optional notice the frontend shows as a toast. Rows a transfer resumed and did not read as on their branch are written Starting, and setup is asked for them once the transfer has ended, so they no longer sit Suspended or Stopped on a running workstation. Push no longer checks that HEAD is on the branch, run now or from the queue: it pushes an explicit refspec. The pull request pipeline keeps the read and goes on when the repair cannot run. Commit, rebase, squash and pull keep the check. No data model changes. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- apps/staged/src-tauri/src/branches.rs | 10 +- .../src/project_transfer/commands.rs | 10 +- .../src/project_transfer/orchestrator.rs | 17 +- .../project_transfer/orchestrator_tests.rs | 24 ++- apps/staged/src-tauri/src/prs/context.rs | 10 +- apps/staged/src-tauri/src/prs/create_pr.rs | 14 +- apps/staged/src-tauri/src/prs/git_pipeline.rs | 10 +- apps/staged/src-tauri/src/prs/push.rs | 11 +- apps/staged/src-tauri/src/prs/tests/push.rs | 57 +++++ apps/staged/src-tauri/src/session_commands.rs | 155 +++++++++++++- .../src-tauri/src/session_queue/retry.rs | 39 ++++ apps/staged/src-tauri/src/workspace_setup.rs | 55 ++++- .../src-tauri/src/workspace_setup/checkout.rs | 142 +++++++++++-- .../src-tauri/src/workspace_setup/clone.rs | 54 ++++- .../src-tauri/src/workspace_setup/failures.rs | 23 ++ .../src-tauri/src/workspace_setup/host.rs | 19 +- .../src-tauri/src/workspace_setup/tests.rs | 16 +- .../src/workspace_setup/tests/checkout.rs | 200 +++++++++++++++++- .../src/workspace_setup/tests/retry.rs | 59 ++++-- .../src/workspace_setup/tests/verify.rs | 20 +- .../src/workspace_setup/tests/wiring.rs | 112 ++++++++-- .../src-tauri/src/workspace_setup/verify.rs | 26 ++- .../projects/workspaceLifecycle.svelte.ts | 11 +- .../projects/workspaceLifecycle.test.ts | 31 ++- 24 files changed, 999 insertions(+), 126 deletions(-) diff --git a/apps/staged/src-tauri/src/branches.rs b/apps/staged/src-tauri/src/branches.rs index b0af7ca85..548752b9a 100644 --- a/apps/staged/src-tauri/src/branches.rs +++ b/apps/staged/src-tauri/src/branches.rs @@ -749,7 +749,7 @@ pub(crate) async fn clone_repo_into_workspace( clone_dir: &str, repo_slug: &str, checkout: &BranchCheckout<'_>, -) -> Result<(), CheckoutError> { +) -> Result, CheckoutError> { let failed = CheckoutError::Failed; let repo_path = resolve_workspace_repo_path(ws_name, clone_dir).map_err(|e| { failed(format!( @@ -808,16 +808,18 @@ pub(crate) async fn check_out_branch_in_workspace( ws_name: &str, remote_dir: Option<&str>, checkout: &BranchCheckout<'_>, -) -> Result<(), CheckoutError> { +) -> Result, CheckoutError> { let workspace = ws_name.to_string(); let remote_dir = remote_dir.map(str::to_string); let base = checkout.base_ref.to_string(); let branch = checkout.branch_name.to_string(); let base_is_worked_on = checkout.base_is_worked_on; + let siblings = checkout.siblings.to_vec(); tauri::async_runtime::spawn_blocking(move || { // Tracked for everything but the reads (`rev-parse`, the // `for-each-ref` that asks whether the clone has the branch, and the - // `log` and `status` that ask what a switch would leave behind): + // `log`, `rev-list` and `status` that ask what a switch would leave + // behind): // Blox SIGKILLs a transient git at 60 s, which a fetch on a large // repo can outlast, and a checkout or fast-forward far from where // the tree is (or smudging LFS files) can outlast it too, leaving @@ -828,6 +830,7 @@ pub(crate) async fn check_out_branch_in_workspace( Some(&"rev-parse") | Some(&"for-each-ref") | Some(&"log") + | Some(&"rev-list") | Some(&"--no-optional-locks") ) { ExecMode::Transient @@ -843,6 +846,7 @@ pub(crate) async fn check_out_branch_in_workspace( base_ref: &base, branch_name: &branch, base_is_worked_on, + siblings: &siblings, }, ) }) diff --git a/apps/staged/src-tauri/src/project_transfer/commands.rs b/apps/staged/src-tauri/src/project_transfer/commands.rs index 47dbf4d32..3b8f27c62 100644 --- a/apps/staged/src-tauri/src/project_transfer/commands.rs +++ b/apps/staged/src-tauri/src/project_transfer/commands.rs @@ -129,7 +129,15 @@ pub(crate) async fn transfer_project_impl( // The guard has dropped either way — unwound, if the task panicked // outside the run's own catch — so what was queued for the transfer // waits on nothing now. - let release = || crate::session_queue::release_held_work(&app_handle, &store, &branch_ids); + // + // A transfer that stopped short may have resumed a workspace and left + // rows `Starting` whose clone it did not read as on their branch + // (`mark_resumed_rows_running`). Setup refused them while the transfer + // ran; it is asked for here, or they would wait for a window to load. + let release = || { + crate::workspace_setup::set_up_branches_left_starting(&app_handle, &store, &branch_ids); + crate::session_queue::release_held_work(&app_handle, &store, &branch_ids) + }; let outcome = match outcome { Ok(outcome) => outcome, Err(e) => { diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs index 8d0a47f10..ed0d3f1e3 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs @@ -527,9 +527,11 @@ impl Run<'_> { /// `Running` means the clone is on the branch as well as the workstation /// being up (`workspace_setup`), and the resume alone vouches only for /// the second. A row whose clone was not read, or read as somewhere - /// else, keeps its status: the status pollers find its workstation - /// running and put it through branch setup, which is what checks the - /// branch out. + /// else, says `Starting`: its workstation is up and its branch is still + /// to be checked out, which is branch setup's to do. Setup leaves a + /// transferring branch alone, so it is asked for once the transfer has + /// ended (`workspace_setup::set_up_branches_left_starting`). Nothing + /// else would: no poller looks at a `Suspended` or `Stopped` row. fn mark_resumed_rows_running( &self, inspection: &Inspection, @@ -540,9 +542,14 @@ impl Run<'_> { .workspace .as_deref() .is_some_and(|workspace| matches!(resumed.get(workspace), Some(None))); - if came_back && row.is_on_its_branch() { + if came_back { + let status = if row.is_on_its_branch() { + WorkspaceStatus::Running + } else { + WorkspaceStatus::Starting + }; self.store - .update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Running)?; + .update_branch_workspace_status(&source.branch.id, &status)?; } } Ok(()) diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs index 76340af70..0a4e028c1 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs @@ -2648,11 +2648,12 @@ fn a_resume_honours_cancel_and_leaves_the_rows_running() { /// A resume vouches for the workstation being up, not for what its clone /// has checked out, and `Running` means both. A row whose clone the -/// transfer's own inspection read as on another branch keeps its status: -/// the status pollers find the workstation running and put the branch -/// through setup, which is what checks it out. +/// transfer's own inspection read as on another branch is left `Starting` +/// when the transfer is blocked, not `Suspended` on a workstation that is +/// up: `Starting` is what branch setup, asked for once the transfer has +/// ended, acts on. #[test] -fn a_resumed_row_whose_clone_is_on_another_branch_is_not_marked_running() { +fn a_resumed_row_whose_clone_is_on_another_branch_is_left_starting() { let f = Fixture::new(&[(WIDGETS, "feature")]); f.transfer(true).result.expect("to remote"); git(&f.checkout(0), &["checkout", "-b", "elsewhere"]); @@ -2668,13 +2669,14 @@ fn a_resumed_row_whose_clone_is_on_another_branch_is_not_marked_running() { .calls() .contains(&format!("resume {}", f.workspace()))); assert!(err.contains("has elsewhere checked out"), "{err}"); - let status = f - .store - .get_branch(&f.branches[0].id) - .unwrap() - .unwrap() - .workspace_status; - assert_eq!(status, Some(WorkspaceStatus::Suspended)); + let row = f.store.get_branch(&f.branches[0].id).unwrap().unwrap(); + assert_eq!(row.workspace_status, Some(WorkspaceStatus::Starting)); + // The transfer is over and the row is one the setup trigger takes. + assert!(!crate::project_transfer::is_branch_transferring(&row.id)); + assert_eq!( + crate::workspace_setup::startup_action(&row), + Some(crate::workspace_setup::StartupAction::SetUp) + ); } // --------------------------------------------------------------------------- diff --git a/apps/staged/src-tauri/src/prs/context.rs b/apps/staged/src-tauri/src/prs/context.rs index 555ad4e6f..f3438a4c7 100644 --- a/apps/staged/src-tauri/src/prs/context.rs +++ b/apps/staged/src-tauri/src/prs/context.rs @@ -103,10 +103,12 @@ pub(crate) fn resolve_branch_pipeline_context( /// A pipeline's share of the checkout check every start on a remote branch /// makes (`workspace_setup::verify`): the clone its git steps run in is read -/// to be on the branch, and put there when it is not. A rebase, squash or -/// commit in a clone that is on the base branch rewrites the base; a push -/// from one sends whatever the local branch last held. A no-op for a local -/// branch. +/// to be on the branch, and put there when it is not. A rebase, squash, +/// commit or pull in a clone that is on the base branch acts on the base. A +/// push does not ask: it names its local branch in an explicit refspec and +/// works from any HEAD. The pull request pipeline asks, so the steps that +/// describe the branch look at it, and goes on whatever the answer. A no-op +/// for a local branch. /// /// Call it once the pipeline is known to run now, before its row goes /// `running`, and keep what it returns until it has diff --git a/apps/staged/src-tauri/src/prs/create_pr.rs b/apps/staged/src-tauri/src/prs/create_pr.rs index a48132496..9465f3a2b 100644 --- a/apps/staged/src-tauri/src/prs/create_pr.rs +++ b/apps/staged/src-tauri/src/prs/create_pr.rs @@ -126,8 +126,20 @@ pub(crate) async fn start_create_pr_pipeline_for_branch( ) -> Result { crate::project_transfer::ensure_branch_not_transferring(&branch_id)?; let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; + // The clone is read and, when it can be, put on the branch, so the + // steps that describe the branch look at it. The pipeline does not fail + // on that: its push names the local branch in an explicit refspec and + // works from any HEAD, a sibling branch with a live session included. // Held until this returns, by when the pipeline's row is `running`. - let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None).await?; + let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None) + .await + .unwrap_or_else(|e| { + log::warn!( + "[create_pr] branch {branch_id}: creating the pull request from a clone that \ + is not on the branch: {e}" + ); + crate::workspace_setup::verify::CheckoutHold::none() + }); let base_branch = base_branch_name(&ctx.branch); let is_draft = draft.unwrap_or(false); diff --git a/apps/staged/src-tauri/src/prs/git_pipeline.rs b/apps/staged/src-tauri/src/prs/git_pipeline.rs index ceb10afe4..f95efdb82 100644 --- a/apps/staged/src-tauri/src/prs/git_pipeline.rs +++ b/apps/staged/src-tauri/src/prs/git_pipeline.rs @@ -92,9 +92,13 @@ pub(crate) async fn start_queued_git_pipeline_for_branch( let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; // Before the claim, like everything else here that awaits. Held until - // this returns, by when the row is claimed `running`. - let _checkout = - ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, Some(&session.id)).await?; + // this returns, by when the row is claimed `running`. Only a pull needs + // the clone on the branch: a push names its local branch in an explicit + // refspec and works from any HEAD. + let _checkout = match queued_pipeline.kind { + Some(PipelineKind::Push) => crate::workspace_setup::verify::CheckoutHold::none(), + _ => ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, Some(&session.id)).await?, + }; let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let RebuiltGitPipeline { steps, diff --git a/apps/staged/src-tauri/src/prs/push.rs b/apps/staged/src-tauri/src/prs/push.rs index cd262f47a..58cd4ad11 100644 --- a/apps/staged/src-tauri/src/prs/push.rs +++ b/apps/staged/src-tauri/src/prs/push.rs @@ -13,9 +13,7 @@ use crate::git; use crate::session_runner; use crate::store::{self, FailureStrategy, PipelineExecution, PipelineKind, PipelineStep, Store}; -use super::context::{ - ensure_pipeline_checkout, resolve_branch_pipeline_context, BranchPipelineContext, -}; +use super::context::{resolve_branch_pipeline_context, BranchPipelineContext}; use super::git_commands::{git_push_with_fallback, push_refspec}; use super::pipeline::{ branch_has_work_in_flight, find_queued_pipeline, launch_running_pipeline_session, @@ -391,9 +389,10 @@ pub(crate) async fn start_or_queue_push_pipeline_for_branch( } let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; - // The branch is idle, so the push runs now. Held until this returns, by - // when its row is `running`. - let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None).await?; + // No check that the clone is on the branch (`ensure_pipeline_checkout`): + // the push names its local branch in an explicit refspec, so it works + // from whatever the clone has checked out, a sibling branch with a live + // session included. let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let steps = build_push_pipeline_steps(&ctx.branch.branch_name, &remote_branch, variant); diff --git a/apps/staged/src-tauri/src/prs/tests/push.rs b/apps/staged/src-tauri/src/prs/tests/push.rs index 6587efe16..98934ac05 100644 --- a/apps/staged/src-tauri/src/prs/tests/push.rs +++ b/apps/staged/src-tauri/src/prs/tests/push.rs @@ -281,3 +281,60 @@ fn deduplicated_pr_branch_pushes_to_the_pr_head_not_its_local_name() { assert!(!command.contains("origin feature-x-2 ")); assert!(!command.contains("feature-x-2:feature-x-2")); } + +/// Why a push makes no check that the clone is on its branch: the step +/// names the local branch in its refspec, so it sends that branch from +/// whatever the clone has checked out. Here that is a sibling branch, as it +/// is while a sibling's session works in a shared clone, where the check +/// used to fail the push. The pull request pipeline's push is the same +/// command. +#[test] +fn a_push_works_from_a_clone_that_has_a_sibling_branch_checked_out() { + use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; + + let seed = TempGitRepo::new(); + seed.write_file("file.txt", "base\n"); + seed.commit("base"); + let origin = TempDir::new("origin"); + let origin_path = origin.path().to_str().unwrap(); + run_git_in(seed.path(), &["clone", "--bare", ".", origin_path]); + let clone = TempDir::new("workstation"); + run_git_in( + origin.path(), + &["clone", origin_path, clone.path().to_str().unwrap()], + ); + run_git_in(clone.path(), &["config", "user.email", "test@example.com"]); + run_git_in(clone.path(), &["config", "user.name", "Test"]); + run_git_in(clone.path(), &["checkout", "-b", "feature-b"]); + std::fs::write(clone.path().join("b.txt"), "b\n").unwrap(); + run_git_in(clone.path(), &["add", "b.txt"]); + run_git_in(clone.path(), &["commit", "-m", "feature-b's work"]); + let pushed = run_git_in(clone.path(), &["rev-parse", "HEAD"]); + run_git_in( + clone.path(), + &["checkout", "-b", "feature-a", "origin/main"], + ); + + let push = build_push_pipeline_steps("feature-b", "feature-b", PUSH); + let pr = build_create_pr_pipeline_steps("pull request", "main", "", "feature-b", "feature-b"); + let (_, command, _) = command_at(&push, 0); + assert_eq!(command_at(&pr, 3).1, command); + let mut sh = std::process::Command::new("sh"); + sh.arg("-c").arg(command).current_dir(clone.path()); + crate::git::strip_git_env(&mut sh); + // No hooks of the machine the test runs on. + sh.env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", "core.hooksPath") + .env("GIT_CONFIG_VALUE_0", "/dev/null"); + let output = sh.output().unwrap(); + assert!(output.status.success(), "{output:?}"); + + assert_eq!( + run_git_in(origin.path(), &["rev-parse", "refs/heads/feature-b"]), + pushed + ); + assert_eq!( + run_git_in(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), + "feature-a" + ); +} diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index 6582d2329..8141c2f12 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -4517,6 +4517,51 @@ fn retry_or_fail_queued_start_before_claim( ) } +/// Hold a remote branch's queue while a session of another branch is at +/// work in the clone the two share, and say whether it is held. +/// +/// A start on this branch would need the clone switched under that session, +/// which branch setup refuses (`workspace_setup::clone`). That is not the +/// row's failure and clears on its own, so the row keeps its place and its +/// attempts, the rows behind it wait with it, and the branch is looked at +/// again shortly: the queue moves once the other session is done. +fn hold_queue_behind_sibling_session( + store: &Store, + kicker: &Arc, + branch: &store::Branch, + session_id: &str, +) -> bool { + use crate::session_queue::retry::QUEUE_HOLD_RECHECK; + + let live = match crate::workspace_setup::clone::sibling_session_in_clone(store, branch) { + Ok(Some(live)) => live, + Ok(None) => return false, + Err(e) => { + log::warn!( + "[session_queue] could not look for sessions sharing the clone of branch {}: {e}", + branch.id + ); + return false; + } + }; + log::info!( + "[session_queue] queued session {session_id} on branch {} waits, and the queue with \ + it: {live}", + branch.id + ); + kicker.start_attempts().hold( + &branch.id, + session_id, + std::time::Instant::now() + QUEUE_HOLD_RECHECK, + ); + kicker.kick_after( + &branch.id, + QUEUE_HOLD_RECHECK, + crate::session_queue::KickSource::StartRetry, + ); + true +} + fn note_session_schedule() -> BranchSessionSchedule { BranchSessionSchedule { kind: BranchSessionScheduleKind::Note, @@ -6411,6 +6456,14 @@ pub(crate) async fn drain_branch_queue( ); break; } + // A row held behind a sibling branch's session is not tried again + // while that session lasts: asked of the database, not the clone. + if kicker.as_ref().is_some_and(|kicker| { + kicker.start_attempts().is_held(&branch_id, &session.id) + && hold_queue_behind_sibling_session(&store, kicker, &branch, &session.id) + }) { + break; + } // The entry gate, re-checked before each claim: every start below // awaits, so a shutdown claimed mid-drain would otherwise keep being @@ -6487,7 +6540,29 @@ pub(crate) async fn drain_branch_queue( // scheduled kick, then end the row with the cause, where // `wait_for_repo_session` and the UI read it, and move // on — an errored row is no barrier. + // + // Two failures are not the row's and spend none of its + // attempts, or every row queued here would be ended in + // turn by a cause none of them has. A start whose check + // of the clone left the branch out of `Running` (a + // repair that failed or was refused): the queue waits + // for the kick `Running` sends, as it does for any + // branch not provisioned. And a start in a clone where + // a sibling branch's session is at work. FailedQueuedStart::StillQueued => { + if branch_session_start_waits_for_provisioning(&store, &branch_id)? { + log::info!( + "[session_queue] queued session {session_id} on branch \ + {branch_id} could not start and the branch is no longer \ + running; its queue waits: {e}" + ); + break; + } + if kicker.as_ref().is_some_and(|kicker| { + hold_queue_behind_sibling_session(&store, kicker, &branch, &session_id) + }) { + break; + } match retry_or_fail_queued_start_before_claim( &store, &app_handle, @@ -6914,8 +6989,10 @@ async fn start_queued_session_for_branch( // local branch; see `workspace_setup::verify`). Before the claim, like // everything else here that awaits: a read that fails leaves the row // queued for the drain's bounded retries, and a repair that fails leaves - // the branch in `Error`, where the queue holds its rows. The clone stays - // held until this returns, by when the row is claimed `running`. + // the branch in `Error`, where the queue holds its rows. A clone a + // sibling branch's session works in is not repaired: the drain holds the + // row while that session lasts and spends no retry on it. The clone + // stays held until this returns, by when the row is claimed `running`. let _checkout = crate::workspace_setup::verify::ensure_session_checkout( &app_handle, &store, @@ -12511,6 +12588,80 @@ mod tests { ); } + /// A queued start in a clone where a sibling branch's session is at work + /// can't have the clone switched under that session. That used to end + /// the row after its three attempts, then the next row the same way. + /// The row is held instead, as often as it is asked about: it stays + /// `queued`, in backoff, with no failure counted, and the branch is + /// kicked again shortly. Once the sibling's session is done nothing + /// holds it, and a start that then fails for a reason of its own counts + /// from one. + #[test] + fn a_queued_row_behind_a_sibling_branchs_session_is_held_without_spending_attempts() { + use crate::session_queue::retry::{ + QueuedStartRetry, MAX_QUEUED_START_ATTEMPTS, QUEUED_START_RETRY_DELAYS, + QUEUE_HOLD_RECHECK, + }; + use crate::session_queue::{BranchQueueKicker, KickSource}; + + let (store, branch) = setup_remote_branch_store(store::WorkspaceStatus::Running); + let mut sibling = + store::Branch::new_remote(&branch.project_id, "sibling", "main", "test-workspace"); + sibling.workspace_status = Some(store::WorkspaceStatus::Running); + store.create_branch(&sibling).unwrap(); + let kicker = Arc::new(BranchQueueKicker::recording()); + let first = store::Session::new_queued("note").with_branch(&branch.id); + store.create_session(&first).unwrap(); + let second = store::Session::new_queued("note").with_branch(&branch.id); + store.create_session(&second).unwrap(); + let hold = || hold_queue_behind_sibling_session(&store, &kicker, &branch, &first.id); + + // Nobody else in the clone: whatever failed is the row's own. + assert!(!hold()); + assert!(kicker.scheduled().is_empty()); + assert!(kicker.start_attempts().is_empty()); + + let agent = store::Session::new_running("work", Path::new("/tmp")).with_branch(&sibling.id); + store.create_session(&agent).unwrap(); + for _ in 0..MAX_QUEUED_START_ATTEMPTS + 2 { + assert!(hold()); + } + let attempts = kicker.start_attempts(); + let now = std::time::Instant::now(); + assert!(attempts.is_held(&branch.id, &first.id)); + assert!(attempts.in_backoff(&branch.id, &first.id, now)); + assert_eq!(attempts.failures(&branch.id, &first.id), 0); + assert_eq!( + kicker.scheduled().last(), + Some(&( + branch.id.clone(), + QUEUE_HOLD_RECHECK, + KickSource::StartRetry + )) + ); + for row in [&first, &second] { + let row = store.get_session(&row.id).unwrap().unwrap(); + assert_eq!(row.status, store::SessionStatus::Queued); + assert_eq!(row.error_message, None); + } + + store + .update_session_status(&agent.id, store::SessionStatus::Completed, None, None) + .unwrap(); + let scheduled = kicker.scheduled().len(); + assert!(!hold()); + assert_eq!(kicker.scheduled().len(), scheduled); + assert!(!attempts.in_backoff(&branch.id, &first.id, now + QUEUE_HOLD_RECHECK * 2)); + assert_eq!( + attempts.record_failure(&branch.id, &first.id, now), + QueuedStartRetry::After { + failures: 1, + delay: QUEUED_START_RETRY_DELAYS[0], + } + ); + assert!(!attempts.is_held(&branch.id, &first.id)); + } + /// A pre-claim start failure is retried before it is recorded: the first /// failures leave the row `queued`, in backoff, with a delayed kick /// scheduled for the branch — nothing else would re-kick it — and the diff --git a/apps/staged/src-tauri/src/session_queue/retry.rs b/apps/staged/src-tauri/src/session_queue/retry.rs index 441f454a4..0b04a27b4 100644 --- a/apps/staged/src-tauri/src/session_queue/retry.rs +++ b/apps/staged/src-tauri/src/session_queue/retry.rs @@ -15,6 +15,11 @@ //! a fresh first attempt anyway. Entries are pruned on every drain to the rows //! still queued on the branch, so a row a Stop or a forced start took leaves //! nothing behind. +//! +//! A start that fails over something that is not the row's failure and will +//! clear on its own is held instead ([`QueuedStartAttempts::hold`]): the +//! row keeps its place and its attempts, and is looked at again after +//! [`QUEUE_HOLD_RECHECK`]. use std::collections::HashMap; use std::sync::Mutex; @@ -53,11 +58,18 @@ pub(crate) fn queued_start_retry(failures: u32) -> QueuedStartRetry { } } +/// How long a held row waits before the drain looks at what holds it again. +/// The look is a database read, so it is short: the queue moves within +/// seconds of the cause going away. +pub(crate) const QUEUE_HOLD_RECHECK: Duration = Duration::from_secs(5); + struct Attempt { failures: u32, /// When the next start may be tried; a kick before then leaves the row /// where it is rather than spending an attempt on it. retry_at: Instant, + /// The row is waiting on something other than its own failed start. + held: bool, } /// Per-row pre-claim start failures, by branch. @@ -86,6 +98,7 @@ impl QueuedStartAttempts { Attempt { failures, retry_at: now + delay, + held: false, }, ); } @@ -99,6 +112,32 @@ impl QueuedStartAttempts { retry } + /// Keep `session_id` where it is until `until` without counting a failed + /// start against it. + pub(crate) fn hold(&self, branch_id: &str, session_id: &str, until: Instant) { + let mut by_branch = self.by_branch.lock().unwrap(); + let rows = by_branch.entry(branch_id.to_string()).or_default(); + let failures = rows.get(session_id).map_or(0, |a| a.failures); + rows.insert( + session_id.to_string(), + Attempt { + failures, + retry_at: until, + held: true, + }, + ); + } + + /// Whether `session_id` was last left where it is by [`Self::hold`]. + pub(crate) fn is_held(&self, branch_id: &str, session_id: &str) -> bool { + self.by_branch + .lock() + .unwrap() + .get(branch_id) + .and_then(|rows| rows.get(session_id)) + .is_some_and(|attempt| attempt.held) + } + /// Whether `session_id` is waiting out the delay from a failed start. pub(crate) fn in_backoff(&self, branch_id: &str, session_id: &str, now: Instant) -> bool { self.by_branch diff --git a/apps/staged/src-tauri/src/workspace_setup.rs b/apps/staged/src-tauri/src/workspace_setup.rs index 47565233b..b5c8b8314 100644 --- a/apps/staged/src-tauri/src/workspace_setup.rs +++ b/apps/staged/src-tauri/src/workspace_setup.rs @@ -269,6 +269,28 @@ pub(crate) fn branch_failed( arm_retry(app_handle, store, branch_id); } +/// Trigger setup for those of `branch_ids` that are remote and in +/// `Starting`: the rows a project transfer left that way when it ended +/// without moving the project. It resumed their workstation and could not +/// read their clone as on the branch, and setup leaves a transferring +/// branch alone until the transfer is over. +pub(crate) fn set_up_branches_left_starting( + app_handle: &tauri::AppHandle, + store: &Arc, + branch_ids: &[String], +) { + for branch_id in branch_ids { + let left_starting = matches!( + store.get_branch(branch_id), + Ok(Some(branch)) if branch.branch_type == store::BranchType::Remote + && startup_action(&branch) == Some(StartupAction::SetUp) + ); + if left_starting { + trigger_branch_setup(app_handle, store, branch_id, false); + } + } +} + /// The startup pass: set up every remote branch a previous run left in /// `Starting`, so recovery doesn't wait for a window to load the project, /// and set the retry timer of every one it left in `Error`. @@ -428,11 +450,18 @@ fn wants_setup(branch: &store::Branch, trigger: SetupTrigger) -> bool { /// Every failure past the first lookups is returned, and leaves the row in /// `Error` with its reason recorded and, unless the checkout refused over /// what is in the clone, a retry timer set. One exception: a repair of a -/// `Running` branch that was refused (a worktree with changes in the way, -/// commits that would be left behind, a live session in the clone) puts the -/// row back to `Running`. The branch was usable before the session that -/// asked came along and is exactly as usable now; only that session fails, -/// with the reason. +/// `Running` branch that was blocked by something outside the clone (a live +/// session working in it, a workstation that is down) puts the row back to +/// `Running`. The clone is fine and the branch is what it was; only the +/// session that asked fails, with the reason. +/// +/// A repair refused over what is in the clone (a worktree with changes in +/// the way, commits that would be left behind) is a failure like any other +/// and leaves `Error`: no session can start on the branch until someone has +/// dealt with the clone, and `Running` would have every queued session ask +/// for the same repair and be ended by the same refusal, one after another. +/// `Error` holds the queue, shows the reason, and the error view's Retry +/// releases it. /// /// Callers go through [`run_guarded`] or [`request_setup`]. pub(crate) async fn set_up_branch( @@ -502,12 +531,15 @@ pub(crate) async fn set_up_branch( let provisioned = provision_and_check_out(guard, store, host, &branch, request, may_start_workstation).await; let error = match provisioned { - Ok(()) => { + Ok(notice) => { let changed = store .update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) .map_err(|e| e.to_string())?; host.status_written(branch_id, changed, &WorkspaceStatus::Running); host.failure_changed(branch_id, None, false); + if let Some(notice) = ¬ice { + host.notice(branch_id, notice); + } guard.forget_retries(branch_id); log::info!( "[workspace_setup] branch={branch_id} checked out '{}' and running", @@ -525,11 +557,12 @@ pub(crate) async fn set_up_branch( let row = store.get_branch(branch_id).ok().flatten(); match row.and_then(|row| row.workspace_status) { Some(WorkspaceStatus::Starting) - if trigger == SetupTrigger::Repair && was_running && error.is_refusal() => + if trigger == SetupTrigger::Repair && was_running && error.is_blocked() => { - // Nothing was touched, so the branch is what it was. The write - // kicks the session queue, which held while the row said - // `Starting`. + // Nothing was touched and nothing is wrong with the clone, so + // the branch is what it was. The write kicks the session queue, + // which held while the row said `Starting` and holds on while a + // sibling's session has the clone (`drain_branch_queue`). if let Ok(changed) = store.update_branch_workspace_status(branch_id, &WorkspaceStatus::Running) { @@ -563,7 +596,7 @@ async fn provision_and_check_out( branch: &store::Branch, request: &SetupRequest, may_start_workstation: bool, -) -> Result<(), CheckoutError> { +) -> Result, CheckoutError> { let failed = CheckoutError::Failed; let checkout = checkout_for(store, branch).map_err(failed)?; let workspace = checkout.workspace.as_str(); diff --git a/apps/staged/src-tauri/src/workspace_setup/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/checkout.rs index 105aa00e2..141fec667 100644 --- a/apps/staged/src-tauri/src/workspace_setup/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/checkout.rs @@ -10,7 +10,10 @@ //! one exception that brings work along on purpose: a clone on the base //! branch, ahead of origin's base, whose branch exists nowhere gets the //! branch created from where it stands, so the commits made there come with -//! it. Local base is left pointing at them, never reset or moved. +//! it. Local base is left pointing at them, never reset or moved. That is +//! done only when the commits can be told to be this branch's +//! ([`refuse_unless_adoptable`]), and the caller is handed a notice to show +//! the user. //! //! A switch that would lose work or move it between branches is //! [`CheckoutError::Refused`], with the clone untouched and a message that @@ -38,10 +41,17 @@ pub(crate) enum CheckoutError { impl CheckoutError { /// Whether the clone was left exactly as it was. + #[cfg(test)] pub(crate) fn is_refusal(&self) -> bool { !matches!(self, CheckoutError::Failed(_)) } + /// Whether something outside the clone was in the way, with the clone + /// itself in order. + pub(crate) fn is_blocked(&self) -> bool { + matches!(self, CheckoutError::Blocked(_)) + } + /// Whether trying again on its own could end differently. Not after a /// refusal over what is in the clone: that waits for someone to deal /// with it and ask again. @@ -83,6 +93,9 @@ pub(crate) struct BranchCheckout<'a> { /// behind on the base by mistake, and they are neither brought along /// nor a reason to refuse. pub(crate) base_is_worked_on: bool, + /// The other branches Staged works on in this clone. Commits stranded + /// on the base could be any of theirs that the clone doesn't have yet. + pub(crate) siblings: &'a [String], } impl<'a> BranchCheckout<'a> { @@ -93,6 +106,7 @@ impl<'a> BranchCheckout<'a> { base_ref, branch_name, base_is_worked_on: false, + siblings: &[], } } } @@ -137,10 +151,13 @@ fn failed(what: impl std::fmt::Display, error: impl std::fmt::Display) -> Checko /// /// Before the clone leaves what it has checked out, [`guard_leaving`] makes /// sure nothing is lost or carried across by the switch. +/// +/// `Ok(Some(notice))` when commits stranded on the base branch were brought +/// along onto the new branch: what the user is to be told. pub(crate) fn check_out_workspace_branch( git: &GitRunner<'_>, checkout: &BranchCheckout<'_>, -) -> Result<(), CheckoutError> { +) -> Result, CheckoutError> { let BranchCheckout { base_ref, branch_name, @@ -202,7 +219,7 @@ pub(crate) fn check_out_workspace_branch( // commits and all, which is the point: never reset it. let _ = git(&["merge", "--ff-only", &remote_branch]); } - return Ok(()); + return Ok(None); } if has_remote_branch { @@ -216,23 +233,29 @@ pub(crate) fn check_out_workspace_branch( &head, branch_name, &["checkout", "-B", branch_name, &remote_branch], - ); + ) + .map(|()| None); } if !stranded.is_empty() { // The branch exists nowhere and the base branch here holds commits // origin's doesn't: work made on the base because the branch was // never checked out. The branch is created where the clone stands, - // so that work is on it. `checkout -b` keeps the worktree as it is - // (it cannot conflict) and leaves local base where it was. - log::warn!( - "[workspace_setup] creating '{branch_name}' from local '{base_ref}', which holds {} \ - commit(s) that are on no other branch and not on origin: {}", + // so that work is on it, once it is clear whose it is. `checkout + // -b` keeps the worktree as it is (it cannot conflict) and leaves + // local base where it was. + refuse_unless_adoptable(git, checkout, &stranded)?; + let notice = format!( + "Branch '{branch_name}' was created from the workstation's local '{base_ref}', \ + which held {} commit(s) that were on no other branch and not on origin: {}. They \ + are on '{branch_name}' now. If they are not this branch's work, remove them from \ + it before you push.", stranded.len(), describe_commits(&stranded) ); + log::warn!("[workspace_setup] {notice}"); return git(&["checkout", "--no-track", "-b", branch_name]) - .map(|_| ()) + .map(|_| Some(notice)) .map_err(|e| failed(format!("could not create branch '{branch_name}'"), e)); } @@ -250,6 +273,7 @@ pub(crate) fn check_out_workspace_branch( &format!("origin/{base_ref}"), ], ) + .map(|()| None) } /// Whether the clone `git` runs in has a local branch `branch_name`. @@ -306,6 +330,7 @@ fn guard_leaving( base_ref, branch_name, base_is_worked_on, + .. } = *checkout; match head { Head::Detached => { @@ -325,16 +350,9 @@ fn guard_leaving( // Commits that only local base holds: on no remote ref and on no // other local branch. Once a branch has been created from them they // are that branch's, and no longer stranded here. - Head::Branch(name) if name == base_ref && !base_is_worked_on => list_commits( - git, - &[ - "HEAD", - "--not", - "--remotes", - &format!("--exclude={base_ref}"), - "--branches", - ], - ), + Head::Branch(name) if name == base_ref && !base_is_worked_on => { + list_commits(git, &stranded_range(&format!("--exclude={base_ref}"))) + } Head::Branch(name) => { let changed = uncommitted_changes(git)?; if changed == 0 { @@ -349,6 +367,90 @@ fn guard_leaving( } } +/// The commits only local base holds, as revision arguments. `exclude_base` +/// is `--exclude=`. +fn stranded_range(exclude_base: &str) -> [&str; 5] { + ["HEAD", "--not", "--remotes", exclude_base, "--branches"] +} + +/// Whether the commits stranded on local base may be brought along onto a +/// new `branch_name`: only when they can be told to be this branch's work. +/// Refused otherwise, with the clone as it was. +/// +/// Two things have to hold. +/// +/// - Every one of them was made in this clone by `git commit`: HEAD's +/// reflog has it as the result of a `commit` entry (an amend included). +/// A commit that reached local base any other way (the clone itself, a +/// pull, a merge, a reset) was on origin's base once. Such commits on no +/// remote ref now are what a force-pushed base leaves behind, and a +/// branch created from them would start from a base origin has dropped. +/// The reflog is the evidence because ancestry is not: a base that was +/// force-pushed and one that simply moved on both leave local base +/// diverged from origin's. +/// - No other branch Staged works on in this clone is missing from it. A +/// branch the clone doesn't have was never checked out here, so its +/// sessions committed on the base as well, and the commits could be its. +fn refuse_unless_adoptable( + git: &GitRunner<'_>, + checkout: &BranchCheckout<'_>, + stranded: &[String], +) -> Result<(), CheckoutError> { + let BranchCheckout { + base_ref, + branch_name, + siblings, + .. + } = *checkout; + let found = format!( + "the clone is on '{base_ref}' with {} commit(s) that are on no other branch and not on \ + origin ({})", + stranded.len(), + describe_commits(stranded) + ); + + let made_here = + git(&["log", "-g", "--grep-reflog=^commit", "--format=%H", "HEAD"]).map_err(|e| { + failed( + "could not read where the clone's local commits came from", + e, + ) + })?; + let shas = git(&[ + &["rev-list"][..], + &stranded_range(&format!("--exclude={base_ref}")), + ] + .concat()) + .map_err(|e| failed("could not list the clone's local commits", e))?; + let made_here = |sha: &str| made_here.lines().any(|line| line.trim() == sha); + if !shas.split_whitespace().all(made_here) { + return Err(CheckoutError::Refused(format!( + "{found}, and not all of them were committed in this clone: they may be what a \ + force-push of '{base_ref}' on origin left behind. They were not put on \ + '{branch_name}' and the clone was left as it is. If they are this branch's work, \ + run `git checkout -b {branch_name}` in the workstation; if not, reset \ + '{base_ref}' to 'origin/{base_ref}'. Then try again." + ))); + } + + let mut missing = Vec::new(); + for sibling in siblings.iter().filter(|name| *name != branch_name) { + if !local_branch_exists(git, sibling).map_err(CheckoutError::Failed)? { + missing.push(format!("'{sibling}'")); + } + } + if !missing.is_empty() { + return Err(CheckoutError::Refused(format!( + "{found}, and the clone has no branch {} either, so they can't be told to be the \ + work of '{branch_name}'. They were not put on it and the clone was left as it \ + is. Put them on the branch they belong to in the workstation (`git checkout -b \ + `), then try again.", + missing.join(" or ") + ))); + } + Ok(()) +} + /// The refusal for a clone on the base branch whose local commits can't be /// brought along by creating the branch, because the branch already exists. fn refuse_stranded( diff --git a/apps/staged/src-tauri/src/workspace_setup/clone.rs b/apps/staged/src-tauri/src/workspace_setup/clone.rs index 97be10662..5d9b26384 100644 --- a/apps/staged/src-tauri/src/workspace_setup/clone.rs +++ b/apps/staged/src-tauri/src/workspace_setup/clone.rs @@ -29,9 +29,12 @@ pub(crate) fn checkout_for(store: &Arc, branch: &store::Branch) -> Result .ok_or("Branch has no workspace name")?; let clone_dir = branches::resolve_branch_clone_dir(store, branch)?; let base_ref = branches::normalize_branch_ref(&branch.base_branch); - let base_is_worked_on = branches_in_clone(store, workspace, clone_dir.as_deref())? - .iter() - .any(|other| other.id != branch.id && other.branch_name == base_ref); + let siblings: Vec = branches_in_clone(store, workspace, clone_dir.as_deref())? + .into_iter() + .filter(|other| other.id != branch.id) + .map(|other| other.branch_name) + .collect(); + let base_is_worked_on = siblings.contains(&base_ref); Ok(Checkout { workspace: workspace.to_string(), clone_dir, @@ -39,6 +42,7 @@ pub(crate) fn checkout_for(store: &Arc, branch: &store::Branch) -> Result base_ref, branch_name: branch.branch_name.clone(), base_is_worked_on, + siblings, }) } @@ -80,13 +84,16 @@ pub(crate) fn branches_in_clone( /// reads as on the branch there is nothing to do, and that read is the /// verification; if not, the checkout is refused rather than run under an /// agent, whose commits would land on the wrong branch. +/// +/// `Ok(Some(notice))` when the checkout brought commits along that the user +/// is to be told about ([`super::checkout::check_out_workspace_branch`]). pub(super) async fn check_out_in_clone( guard: &Arc, store: &Arc, host: &dyn SetupHost, checkout: &Checkout, request: &SetupRequest, -) -> Result<(), CheckoutError> { +) -> Result, CheckoutError> { let failed = CheckoutError::Failed; // Before the gate: this can wait on the reconcile service. settle_disconnected_sessions(store, host, checkout, request).await; @@ -97,7 +104,7 @@ pub(super) async fn check_out_in_clone( if let Some(live) = live_session_in_clone(store, checkout, request).map_err(failed)? { let (workspace, expected) = (&checkout.workspace, &checkout.branch_name); return match verify::read_head(host, checkout).await { - Ok(head) if verify::is_on_branch(&head, checkout) => Ok(()), + Ok(head) if verify::is_on_branch(&head, checkout) => Ok(None), Ok(head) => Err(CheckoutError::Blocked(format!( "Workspace '{workspace}' has {head} checked out, where branch '{expected}' was \ expected. It was not switched because {live}; try again once that session has \ @@ -107,13 +114,14 @@ pub(super) async fn check_out_in_clone( }; } - host.check_out(checkout).await?; + let notice = host.check_out(checkout).await?; // The checkout said it worked. `Running` rests on where the clone is, // so that is read, not inferred. verify::confirm_checked_out(host, checkout) .await - .map_err(failed) + .map_err(failed)?; + Ok(notice) } /// A session found working in a clone. @@ -193,6 +201,38 @@ pub(crate) fn live_session_in_clone( Ok(live(false).or_else(|| live(true))) } +/// A live session of another branch in `branch`'s clone: what a repair of +/// `branch` would be blocked by, for as long as it lasts. Read from the +/// database alone, so the session queue can ask whether a branch it holds +/// over one is still held ([`crate::session_commands::drain_branch_queue`]). +pub(crate) fn sibling_session_in_clone( + store: &Store, + branch: &store::Branch, +) -> Result, String> { + let Some(workspace) = branch.workspace_name.as_deref() else { + return Ok(None); + }; + if branch.branch_type != store::BranchType::Remote { + return Ok(None); + } + let clone_dir = branches::resolve_branch_clone_dir(store, branch)?; + for sibling in branches_in_clone(store, workspace, clone_dir.as_deref())? { + if sibling.id == branch.id { + continue; + } + let busy = store + .get_busy_sessions_for_branch(&sibling.id) + .map_err(|e| e.to_string())?; + if let Some(session) = busy.first() { + return Ok(Some(LiveSession { + branch_name: sibling.branch_name, + disconnected: session.status == store::SessionStatus::Disconnected, + })); + } + } + Ok(None) +} + /// Give the `disconnected` sessions in `checkout`'s clone a chance to be /// settled before they count as live. /// diff --git a/apps/staged/src-tauri/src/workspace_setup/failures.rs b/apps/staged/src-tauri/src/workspace_setup/failures.rs index 14cb19504..f760fa3ea 100644 --- a/apps/staged/src-tauri/src/workspace_setup/failures.rs +++ b/apps/staged/src-tauri/src/workspace_setup/failures.rs @@ -17,6 +17,10 @@ //! fail into the card only, and a reason that has not changed is not //! announced again. //! +//! The same event carries the one thing a setup that worked has to say: a +//! `notice` that commits found on the base branch were brought along onto +//! the new branch ([`notify`]). It is not a reason and is not kept. +//! //! In memory, per process: the branch row keeps only its status. After a //! restart the reasons are gone until the next attempt fails, which the //! startup pass's retries ([`super::retry`]) bring about on their own. @@ -37,6 +41,8 @@ struct SetupErrorEvent<'a> { branch_id: &'a str, error: Option<&'a str>, user_requested: bool, + #[serde(skip_serializing_if = "Option::is_none")] + notice: Option<&'a str>, } fn reasons() -> &'static Mutex> { @@ -63,6 +69,7 @@ pub(crate) fn set( branch_id, error, user_requested: user_requested && error.is_some(), + notice: None, }; crate::web_server::emit_to_all(app_handle, EVENT, event); }; @@ -84,6 +91,22 @@ pub(crate) fn set( } } +/// Tell every window and web client something about a setup of `branch_id` +/// that worked. The branch has no reason to show, so `error` is `null`. +pub(crate) fn notify( + app_handle: &tauri::AppHandle, + branch_id: &str, + notice: &str, +) { + let event = SetupErrorEvent { + branch_id, + error: None, + user_requested: false, + notice: Some(notice), + }; + crate::web_server::emit_to_all(app_handle, EVENT, event); +} + fn lock() -> std::sync::MutexGuard<'static, HashMap> { reasons().lock().unwrap_or_else(|p| p.into_inner()) } diff --git a/apps/staged/src-tauri/src/workspace_setup/host.rs b/apps/staged/src-tauri/src/workspace_setup/host.rs index f7ba8f027..56610da22 100644 --- a/apps/staged/src-tauri/src/workspace_setup/host.rs +++ b/apps/staged/src-tauri/src/workspace_setup/host.rs @@ -60,6 +60,9 @@ pub(crate) struct Checkout { /// Whether another branch in this clone is the base branch itself (see /// [`BranchCheckout::base_is_worked_on`]). pub(crate) base_is_worked_on: bool, + /// The other branches whose checkout is this clone (see + /// [`BranchCheckout::siblings`]). + pub(crate) siblings: Vec, } impl Checkout { @@ -68,6 +71,7 @@ impl Checkout { base_ref: &self.base_ref, branch_name: &self.branch_name, base_is_worked_on: self.base_is_worked_on, + siblings: &self.siblings, } } } @@ -88,8 +92,9 @@ pub(crate) trait SetupHost: Send + Sync + 'static { async fn start_workspace(&self, workspace: &str, source: &str) -> Result<(), BloxError>; /// Make sure the clone exists and put it on the branch - /// ([`checkout::check_out_workspace_branch`]). - async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError>; + /// ([`checkout::check_out_workspace_branch`], whose notice this hands + /// on). + async fn check_out(&self, checkout: &Checkout) -> Result, CheckoutError>; /// What the clone has checked out: one cheap read, no fetch /// ([`verify::read_clone_head`]). `Err` when the read didn't answer, @@ -105,6 +110,10 @@ pub(crate) trait SetupHost: Send + Sync + 'static { /// more ([`failures`]). fn failure_changed(&self, branch_id: &str, error: Option<&str>, user_asked: bool); + /// Something a setup that worked has to tell the user: commits it + /// brought along onto the branch ([`failures::notify`]). + fn notice(&self, branch_id: &str, notice: &str); + /// Call [`retry::fire`] for `branch_id` with `ticket` once `delay` has /// passed. fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64); @@ -177,7 +186,7 @@ impl SetupHost for BloxHost { .map(|_| ()) } - async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError> { + async fn check_out(&self, checkout: &Checkout) -> Result, CheckoutError> { match checkout.clone_dir.as_deref() { Some(clone_dir) => { branches::clone_repo_into_workspace( @@ -217,6 +226,10 @@ impl SetupHost for BloxHost { failures::set(&self.app_handle, &self.store, branch_id, error, user_asked); } + fn notice(&self, branch_id: &str, notice: &str) { + failures::notify(&self.app_handle, branch_id, notice); + } + fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { let app_handle = self.app_handle.clone(); let store = Arc::clone(&self.store); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests.rs b/apps/staged/src-tauri/src/workspace_setup/tests.rs index 5e69332e2..f6e907428 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests.rs @@ -171,6 +171,8 @@ struct FakeHost { failures: Mutex)>>, /// The branches whose failure was announced as one the user asked for. user_asked_failures: Mutex>, + /// What setups that worked told the user: branch, notice. + notices: Mutex>, /// Every retry timer asked for: branch, delay, ticket. Nothing sleeps; /// a test fires the ones it wants. retries: Mutex>, @@ -198,6 +200,7 @@ impl FakeHost { written: Mutex::new(Vec::new()), failures: Mutex::new(Vec::new()), user_asked_failures: Mutex::new(Vec::new()), + notices: Mutex::new(Vec::new()), retries: Mutex::new(Vec::new()), on_check_disconnected: None, checked_sessions: Mutex::new(Vec::new()), @@ -248,6 +251,10 @@ impl FakeHost { .and_then(|(_, reason)| reason) } + fn notices(&self) -> Vec<(String, String)> { + self.notices.lock().unwrap().clone() + } + fn user_asked_failures(&self) -> Vec { self.user_asked_failures.lock().unwrap().clone() } @@ -321,7 +328,7 @@ impl SetupHost for FakeHost { } } - async fn check_out(&self, checkout: &Checkout) -> Result<(), CheckoutError> { + async fn check_out(&self, checkout: &Checkout) -> Result, CheckoutError> { self.calls.lock().unwrap().push("check_out"); if let Some(gate) = &self.gate { gate.acquire().await.unwrap().forget(); @@ -372,6 +379,13 @@ impl SetupHost for FakeHost { } } + fn notice(&self, branch_id: &str, notice: &str) { + self.notices + .lock() + .unwrap() + .push((branch_id.to_string(), notice.to_string())); + } + fn retry_later(&self, branch_id: &str, delay: Duration, ticket: u64) { self.retries .lock() diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs index c016ba616..4dd519049 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs @@ -7,13 +7,14 @@ use std::sync::Arc; use super::super::checkout::{local_branch_exists, BranchCheckout, CheckoutError}; use super::super::SetupTrigger; +use super::verify::running_branch; use super::{ head_branch, origin_and_workstation_clone, set_up, starting_branch, status_of, store_with_project, FakeHost, Info, Start, }; use crate::branches; use crate::store::WorkspaceStatus; -use crate::test_utils::run_git_in; +use crate::test_utils::{run_git_in, TempDir}; fn rev(clone: &Path, name: &str) -> String { run_git_in(clone, &["rev-parse", name]).trim().to_string() @@ -39,12 +40,12 @@ fn has_branch(clone: &Path, name: &str) -> bool { local_branch_exists(&branches::git_runner(clone, None, None), name).unwrap() } -fn check_out(clone: &Path, checkout: &BranchCheckout<'_>) -> Result<(), CheckoutError> { +fn check_out(clone: &Path, checkout: &BranchCheckout<'_>) -> Result, CheckoutError> { branches::check_out_workspace_branch(&branches::git_runner(clone, None, None), checkout) } /// A refusal, as the text the user reads. -fn refusal(result: Result<(), CheckoutError>) -> String { +fn refusal(result: Result, CheckoutError>) -> String { let error = result.unwrap_err(); assert!(error.is_refusal(), "not a refusal: {error}"); error.to_string() @@ -55,7 +56,8 @@ fn refusal(result: Result<(), CheckoutError>) -> String { /// The state the original bug left behind: the agent ran in a clone still on /// `main` and committed there. The branch exists nowhere, so it is created /// where the clone stands and the commits are on it. Local `main` is not -/// reset or moved, and the new branch does not track `origin/main`. +/// reset or moved, and the new branch does not track `origin/main`. The +/// caller is handed a notice naming the commits, for the user. #[test] fn commits_stranded_on_the_base_branch_come_along_to_a_new_branch() { let (_origin, clone) = origin_and_workstation_clone(); @@ -64,7 +66,19 @@ fn commits_stranded_on_the_base_branch_come_along_to_a_new_branch() { commit(clone, "one.txt", "agent work one"); let stranded = commit(clone, "two.txt", "agent work two"); - check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + let notice = check_out(clone, &BranchCheckout::new("main", "feature-new")) + .unwrap() + .expect("a notice that commits were brought along"); + + for part in [ + "'feature-new'", + "local 'main'", + "2 commit(s)", + "agent work one", + "agent work two", + ] { + assert!(notice.contains(part), "{part}: {notice}"); + } assert_eq!(head_branch(clone), "feature-new"); assert_eq!(rev(clone, "HEAD"), stranded); @@ -110,7 +124,11 @@ fn a_later_branch_starts_from_origins_base_not_from_the_brought_along_commits() let stranded = commit(clone, "one.txt", "agent work"); check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); - check_out(clone, &BranchCheckout::new("main", "feature-second")).unwrap(); + // Nothing was brought along, so there is nothing to tell the user. + assert_eq!( + check_out(clone, &BranchCheckout::new("main", "feature-second")), + Ok(None) + ); assert_eq!(head_branch(clone), "feature-second"); assert_eq!(rev(clone, "HEAD"), origin_main); @@ -192,6 +210,7 @@ fn commits_on_a_base_branch_that_is_worked_on_stay_on_it() { base_ref: "main", branch_name: "feature-child", base_is_worked_on: true, + siblings: &[], }; std::fs::write(clone.join("file.txt"), "base\nin progress\n").unwrap(); @@ -211,6 +230,123 @@ fn commits_on_a_base_branch_that_is_worked_on_stay_on_it() { assert_eq!(rev(clone, "main"), parents_own); } +// ── Commits on the base branch that can't be told to be this branch's ─────── + +/// Two branches of one repository share the clone and neither was ever +/// checked out in it, so the commits on `main` could be either's. The first +/// to be set up does not take them: the checkout is refused, naming the +/// other branch, and the clone is untouched. Once the clone has the other +/// branch, that branch holds its own commits and what is left on `main` +/// has one possible owner. +#[test] +fn stranded_commits_another_branch_could_own_are_not_brought_along() { + let (_origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let stranded = commit(clone, "one.txt", "agent work"); + let siblings = ["feature-a".to_string()]; + let checkout = BranchCheckout { + base_ref: "main", + branch_name: "feature-b", + base_is_worked_on: false, + siblings: &siblings, + }; + + let error = refusal(check_out(clone, &checkout)); + + for part in [ + "the clone is on 'main'", + "agent work", + "no branch 'feature-a' either", + "'feature-b'", + ] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(head_branch(clone), "main"); + assert_eq!(rev(clone, "main"), stranded); + assert!(!has_branch(clone, "feature-b")); + + run_git_in(clone, &["branch", "feature-a", "origin/main"]); + let notice = check_out(clone, &checkout).unwrap(); + assert!(notice.is_some_and(|notice| notice.contains("agent work"))); + assert_eq!(head_branch(clone), "feature-b"); + assert_eq!(rev(clone, "HEAD"), stranded); +} + +/// A clone of `origin` that stands for everyone else pushing to it. +fn upstream_clone(origin: &Path) -> TempDir { + let upstream = TempDir::new("upstream"); + let (from, to) = (origin.to_str().unwrap(), upstream.path().to_str().unwrap()); + run_git_in(origin, &["clone", from, to]); + run_git_in(upstream.path(), &["config", "user.email", "up@example.com"]); + run_git_in(upstream.path(), &["config", "user.name", "Upstream"]); + upstream +} + +/// Origin's base was force-pushed after the clone had pulled it. The +/// commits origin dropped are still on local `main`, on no remote ref and +/// no other branch, exactly like commits an agent made there. They were not +/// committed in this clone, which its reflog shows, so a new branch is not +/// created from them, with or without an agent's commit on top. Once local +/// `main` is reset the branch starts from origin's base. +#[test] +fn commits_a_force_pushed_base_left_behind_are_not_brought_along() { + let (origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let upstream = upstream_clone(origin.path()); + commit(upstream.path(), "up.txt", "upstream work"); + run_git_in(upstream.path(), &["push", "origin", "main"]); + run_git_in(clone, &["pull", "--ff-only", "origin", "main"]); + run_git_in(upstream.path(), &["reset", "--hard", "HEAD~1"]); + commit(upstream.path(), "other.txt", "rewritten base"); + run_git_in(upstream.path(), &["push", "--force", "origin", "main"]); + let left_behind = rev(clone, "main"); + let checkout = BranchCheckout::new("main", "feature-new"); + + let error = refusal(check_out(clone, &checkout)); + + for part in [ + "upstream work", + "not all of them were committed in this clone", + "force-push of 'main'", + "'feature-new'", + ] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(head_branch(clone), "main"); + assert_eq!(rev(clone, "main"), left_behind); + assert!(!has_branch(clone, "feature-new")); + + let on_top = commit(clone, "one.txt", "agent work"); + let error = refusal(check_out(clone, &checkout)); + assert!(error.contains("2 commit(s)"), "{error}"); + assert_eq!(rev(clone, "main"), on_top); + assert!(!has_branch(clone, "feature-new")); + + run_git_in(clone, &["reset", "--hard", "origin/main"]); + assert_eq!(check_out(clone, &checkout), Ok(None)); + assert_eq!(head_branch(clone), "feature-new"); + assert_eq!(rev(clone, "HEAD"), rev(clone, "origin/main")); +} + +/// A base that only moved on, with no force-push, leaves local `main` +/// behind origin's and the agent's commits on top of the old tip. Those +/// were committed here and still come along. +#[test] +fn commits_made_here_come_along_when_origins_base_has_moved_on() { + let (origin, clone) = origin_and_workstation_clone(); + let clone = clone.path(); + let stranded = commit(clone, "one.txt", "agent work"); + let upstream = upstream_clone(origin.path()); + commit(upstream.path(), "up.txt", "upstream work"); + run_git_in(upstream.path(), &["push", "origin", "main"]); + + let notice = check_out(clone, &BranchCheckout::new("main", "feature-new")).unwrap(); + + assert!(notice.is_some_and(|notice| notice.contains("1 commit(s)"))); + assert_eq!(head_branch(clone), "feature-new"); + assert_eq!(rev(clone, "HEAD"), stranded); +} + // ── Other states the clone can't leave ────────────────────────────────────── /// Uncommitted changes on another branch are that branch's work. Git would @@ -287,6 +423,58 @@ async fn setup_brings_the_commits_an_agent_left_on_the_base_branch() { assert_eq!(rev(clone.path(), "HEAD"), stranded); assert_eq!(rev(clone.path(), "main"), stranded); assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + // The user is told, not only the log. + let notices = host.notices(); + assert_eq!(notices.len(), 1, "{notices:?}"); + assert_eq!(notices[0].0, branch.id); + assert!(notices[0].1.contains("agent work"), "{notices:?}"); +} + +/// The incident with a second branch in the clone. `feature-a` was marked +/// `Running` over a clone still on `main` and its agent committed there. +/// `feature-b` is added and set up first. It used to be created from `main` +/// with the other branch's work on it; now its setup fails where the user +/// can see why, with no timer, and nothing is moved. Once the commits are +/// on their branch in the workstation, `feature-b` starts from origin's +/// base without them. +#[tokio::test] +async fn two_branches_that_could_own_stranded_commits_are_not_guessed_between() { + let (_origin, clone) = origin_and_workstation_clone(); + let origin_main = rev(clone.path(), "origin/main"); + let stranded = commit(clone.path(), "one.txt", "agent work"); + let (store, project) = store_with_project(); + let _first = running_branch(&store, &project, "feature-a"); + let second = starting_branch(&store, &project, "feature-b"); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + let error = set_up(&store, &host, &second, SetupTrigger::Requested) + .await + .unwrap_err(); + + for part in ["agent work", "no branch 'feature-a' either", "'feature-b'"] { + assert!(error.contains(part), "{part}: {error}"); + } + assert_eq!(head_branch(clone.path()), "main"); + assert!(!has_branch(clone.path(), "feature-b")); + assert_eq!(status_of(&store, &second), Some(WorkspaceStatus::Error)); + assert_eq!(host.failure_of(&second), Some(error)); + assert_eq!(host.retries(), vec![]); + assert_eq!(host.notices(), vec![]); + + run_git_in(clone.path(), &["checkout", "-b", "feature-a"]); + set_up(&store, &host, &second, SetupTrigger::Requested) + .await + .unwrap(); + + assert_eq!(head_branch(clone.path()), "feature-b"); + assert_eq!(rev(clone.path(), "HEAD"), origin_main); + assert_eq!(rev(clone.path(), "feature-a"), stranded); + assert_eq!(status_of(&store, &second), Some(WorkspaceStatus::Running)); + assert_eq!(host.notices(), vec![]); } /// A branch that was never `Running` and can't be checked out without diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs index f1be8e0eb..c7d9b2cb2 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/retry.rs @@ -1,7 +1,8 @@ //! `Error` has a way out: every failure records why and sets a retry, the //! retry sets the branch up once its workstation is running without the -//! card noticing until it has, and a refused repair does not turn a branch -//! that was `Running` into a failed one. +//! card noticing until it has. A repair refused over what is in the clone +//! fails the branch like any setup, with no timer; one blocked by something +//! outside the clone leaves a `Running` branch `Running`. use std::sync::Arc; use std::time::Duration; @@ -325,12 +326,15 @@ async fn a_branch_seen_failed_gets_one_timer() { assert_eq!(host.retries().len(), 2); } -/// A repair that is refused changed nothing in the clone, so a branch that -/// was `Running` is `Running` again: only the session that asked fails, -/// with the reason. Here the clone is on another branch with uncommitted -/// changes, which a checkout would carry across. +/// A repair refused over what is in the clone changed nothing in it, but no +/// session can start on the branch until someone has dealt with the clone. +/// The branch goes to `Error` with the reason and no timer, where the +/// session queue holds its rows instead of ending them one after another on +/// the same refusal, and the error view's Retry lets them go. The session +/// that asked fails with the reason. Here the clone is on another branch +/// with uncommitted changes, which a checkout would carry across. #[tokio::test] -async fn a_refused_repair_leaves_a_running_branch_running() { +async fn a_repair_refused_over_the_clone_fails_the_branch_with_no_timer() { let (_origin, clone) = origin_and_workstation_clone(); let (store, project) = store_with_project(); let branch = running_branch(&store, &project, "feature-b"); @@ -349,23 +353,37 @@ async fn a_refused_repair_leaves_a_running_branch_running() { ] { assert!(error.contains(part), "{part}: {error}"); } - assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); - assert_eq!(host.failure_of(&branch), None); - assert_eq!(host.retries(), vec![]); - // The row went through `Starting`; the write back kicks the queue that - // held meanwhile. - assert_eq!( - host.written(), - vec![(branch.id.clone(), true, WorkspaceStatus::Running)] + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + let reason = host.failure_of(&branch).expect("a reason on the card"); + assert!( + reason.contains("uncommitted changes to 1 file(s)"), + "{reason}" ); + assert_eq!(host.retries(), vec![]); + assert_eq!(host.written(), vec![]); assert_eq!(head_branch(clone.path()), "feature-a"); let dirty = run_git_in(clone.path(), &["status", "--porcelain"]); assert!(dirty.contains("file.txt"), "{dirty}"); + + // The changes are committed where they belong and the user retries. + run_git_in(clone.path(), &["commit", "-am", "feature-a's work"]); + set_up_under( + &Arc::new(SetupGuard::default()), + &store, + &host, + &branch, + SetupTrigger::Requested, + ) + .await + .unwrap(); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + assert_eq!(head_branch(clone.path()), "feature-b"); + assert_eq!(host.failure_of(&branch), None); } /// The same for commits a switch would leave behind on the base branch. #[tokio::test] -async fn a_repair_refused_over_stranded_commits_leaves_the_branch_running() { +async fn a_repair_refused_over_stranded_commits_fails_the_branch() { let (_origin, clone) = origin_and_workstation_clone(); let (store, project) = store_with_project(); let branch = running_branch(&store, &project, "feature-b"); @@ -380,16 +398,17 @@ async fn a_repair_refused_over_stranded_commits_leaves_the_branch_running() { for part in ["'main'", "'feature-b'", "agent work"] { assert!(error.contains(part), "{part}: {error}"); } - assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); - assert_eq!(host.failure_of(&branch), None); + assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Error)); + assert!(host.failure_of(&branch).is_some()); assert_eq!(host.retries(), vec![]); assert_eq!(head_branch(clone.path()), "main"); let main = run_git_in(clone.path(), &["rev-parse", "main"]); assert_eq!(main.trim(), stranded); } -/// A repair that finds the workstation down is refused too: what the row of -/// a stopped workstation should say is the pollers' to settle. +/// A repair that finds the workstation down is blocked by something outside +/// the clone and leaves the row `Running`: what the row of a stopped +/// workstation should say is the pollers' to settle. #[tokio::test] async fn a_repair_never_starts_a_workstation_for_a_running_row() { let (_origin, clone) = origin_and_workstation_clone(); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs index d67bfbf9b..c6ebdf350 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/verify.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use super::super::checkout::{local_branch_exists, BranchCheckout}; +use super::super::clone::sibling_session_in_clone; use super::super::guard::SetupGuard; use super::super::host::SetupHost; use super::super::verify::{parse_clone_head, verify_session_checkout, CloneHead}; @@ -339,8 +340,11 @@ async fn a_session_start_reads_once_more_after_an_unanswered_read() { /// Two branches of one repository share a clone. While a session of one is /// live in it the clone is not switched under that agent: the other -/// branch's session fails, naming both. Once the clone is free the same -/// start repairs it. +/// branch's session, or pipeline, fails, naming both and the cause. The +/// database already says so, so no repair is run for it: one read of the +/// clone, and the row never leaves `Running`. The session queue holds the +/// branch's queued rows on the same fact ([`sibling_session_in_clone`]). +/// Once the clone is free the same start repairs it. #[tokio::test] async fn a_clone_another_live_session_works_in_is_not_switched() { let (_origin, clone) = origin_and_workstation_clone(); @@ -358,9 +362,18 @@ async fn a_clone_another_live_session_works_in_is_not_switched() { assert!(error.contains("branch 'feature-a'"), "{error}"); assert!(error.contains("'feature-b'"), "{error}"); assert!(error.contains("running in the same clone"), "{error}"); - assert_eq!(host.count("check_out"), 0); + assert_eq!(calls(&host), vec!["clone_head"]); + assert_eq!(host.written(), vec![]); + assert_eq!(host.failures(), vec![]); assert_eq!(head_branch(clone.path()), "feature-a"); assert_eq!(status_of(&store, &branch), Some(WorkspaceStatus::Running)); + let held_by = sibling_session_in_clone(&store, &branch).unwrap(); + assert_eq!( + held_by.map(|live| live.branch_name).as_deref(), + Some("feature-a") + ); + // The sibling is not held by its own session. + assert_eq!(sibling_session_in_clone(&store, &sibling), Ok(None)); store .update_session_status( @@ -370,6 +383,7 @@ async fn a_clone_another_live_session_works_in_is_not_switched() { None, ) .unwrap(); + assert_eq!(sibling_session_in_clone(&store, &branch), Ok(None)); start_session_on(&store, &host, &branch, "session-b") .await .unwrap(); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index 0824934fa..6d45487f5 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -24,6 +24,7 @@ const PR_PUSH: &str = include_str!("../../prs/push.rs"); const PR_CREATE: &str = include_str!("../../prs/create_pr.rs"); const PR_PULL: &str = include_str!("../../prs/pull.rs"); const TRANSFER: &str = include_str!("../../project_transfer/orchestrator.rs"); +const TRANSFER_COMMANDS: &str = include_str!("../../project_transfer/commands.rs"); /// The text of the function whose signature starts with `signature`, to the /// closing brace at its own indentation. @@ -290,7 +291,7 @@ fn status_writes<'a>(source: &'a str, write: &str) -> Vec<&'a str> { /// Outside setup, nothing in the commands, the pollers, the session starts /// or the web server writes a literal `Running`. Setup writes it in two /// places: after a verified checkout, and when it puts back a `Running` -/// row whose repair was refused. +/// row whose repair was blocked by something outside the clone. #[test] fn only_setup_writes_running() { for (name, source) in [ @@ -336,7 +337,10 @@ fn only_setup_writes_running() { /// A transfer that resumed a source workspace does not mark its rows /// `Running` wholesale. It marks the ones its own inspection read as on -/// their branch, after that inspection. +/// their branch, after that inspection, and leaves the rest `Starting`. +/// Setup refuses a transferring branch, so the command asks for it once +/// the transfer has ended without moving the project, before anything held +/// for the transfer is released. #[test] fn a_transfer_marks_running_only_the_rows_it_read_on_their_branch() { assert!( @@ -347,8 +351,25 @@ fn a_transfer_marks_running_only_the_rows_it_read_on_their_branch() { assert_before( "mark_resumed_rows_running", mark, - "row.is_on_its_branch()", - "update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Running)", + "let status = if row.is_on_its_branch() {\n WorkspaceStatus::Running\n } else {\n WorkspaceStatus::Starting\n };", + "update_branch_workspace_status(&source.branch.id, &status)", + ); + assert_eq!(mark.matches("WorkspaceStatus::Running").count(), 1); + let transfer = body_of( + TRANSFER_COMMANDS, + "pub(crate) async fn transfer_project_impl(", + ); + assert_before( + "transfer_project_impl", + transfer, + "crate::workspace_setup::set_up_branches_left_starting(&app_handle, &store, &branch_ids);", + "crate::session_queue::release_held_work(&app_handle, &store, &branch_ids)", + ); + let left_starting = body_of(SETUP, "pub(crate) fn set_up_branches_left_starting<"); + assert_calls( + "set_up_branches_left_starting", + left_starting, + "trigger_branch_setup(app_handle, store, branch_id, false);", ); assert_eq!( TRANSFER.matches("update_branch_workspace_status(").count(), @@ -428,9 +449,13 @@ fn every_session_start_checks_the_clone_before_it_launches() { ); } -/// The pipelines (commit, rebase, squash, push, pull request) run git in -/// the clone without an agent, and get the same check before their first -/// step: before the row is claimed or inserted as running. +/// The pipelines that act on HEAD (commit, rebase, squash, pull) run git +/// in the clone without an agent, and get the same check before their first +/// step: before the row is claimed or inserted as running. The pull request +/// pipeline makes it too, for the steps that describe the branch, and goes +/// on whatever it says. A push does not make it: it names its local branch +/// in an explicit refspec, and used to fail while a sibling branch's +/// session had the clone. #[test] fn every_pipeline_checks_the_clone_before_its_git_steps() { let starts = [ @@ -449,11 +474,6 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { "pub(crate) async fn start_queued_git_pipeline_for_branch(", "claim.claim(&store)", ), - ( - PR_PUSH, - "pub(crate) async fn start_or_queue_push_pipeline_for_branch(", - "insert_running_push_pipeline_session(", - ), ( PR_CREATE, "pub(crate) async fn start_create_pr_pipeline_for_branch(", @@ -474,6 +494,35 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { ); } + // A push is not held up by where the clone is, run now or from the + // queue, and the pull request pipeline never fails on it. + let push = body_of( + PR_PUSH, + "pub(crate) async fn start_or_queue_push_pipeline_for_branch(", + ); + assert!( + !push.contains("ensure_pipeline_checkout(&"), + "a push must not check the clone's HEAD" + ); + let queued = body_of( + PR_GIT_PIPELINE, + "pub(crate) async fn start_queued_git_pipeline_for_branch(", + ); + assert_before( + "start_queued_git_pipeline_for_branch", + queued, + "Some(PipelineKind::Push) => crate::workspace_setup::verify::CheckoutHold::none(),", + "ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, Some(&session.id)).await?", + ); + let create_pr = body_of( + PR_CREATE, + "pub(crate) async fn start_create_pr_pipeline_for_branch(", + ); + assert!( + !create_pr.contains(".await?") && create_pr.contains(".unwrap_or_else(|e| {"), + "the pull request pipeline must not fail on the checkout check" + ); + // The pull: checked in the task that runs it, before the pull itself. let pull = body_of( PR_PULL, @@ -499,3 +548,42 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { "crate::workspace_setup::verify::ensure_session_checkout(", ); } + +// ── What holds a branch's queue behind a start that could not check out ───── + +/// A queued start that fails over where the clone is spends none of the +/// row's attempts: the drain breaks off when the start left the branch out +/// of `Running`, and when a sibling branch's session has the clone, before +/// it counts a failure. A held row is asked about again from the database, +/// before it is started. +#[test] +fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { + let drain = body_of(SESSION_COMMANDS, "pub(crate) async fn drain_branch_queue("); + let still_queued = &drain[drain + .rfind("FailedQueuedStart::StillQueued => {") + .expect("the pre-claim failure arm")..]; + assert_before( + "drain_branch_queue", + still_queued, + "if branch_session_start_waits_for_provisioning(&store, &branch_id)? {", + "retry_or_fail_queued_start_before_claim(", + ); + assert_before( + "drain_branch_queue", + still_queued, + "hold_queue_behind_sibling_session(&store, kicker, &branch, &session_id)", + "retry_or_fail_queued_start_before_claim(", + ); + assert_before( + "drain_branch_queue", + drain, + "kicker.start_attempts().is_held(&branch_id, &session.id)\n && hold_queue_behind_sibling_session(&store, kicker, &branch, &session.id)", + "start_queued_session_for_branch(", + ); + let hold = body_of(SESSION_COMMANDS, "fn hold_queue_behind_sibling_session("); + assert_calls( + "hold_queue_behind_sibling_session", + hold, + "crate::workspace_setup::clone::sibling_session_in_clone(store, branch)", + ); +} diff --git a/apps/staged/src-tauri/src/workspace_setup/verify.rs b/apps/staged/src-tauri/src/workspace_setup/verify.rs index fd12ff2cd..d8d72de22 100644 --- a/apps/staged/src-tauri/src/workspace_setup/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/verify.rs @@ -25,10 +25,12 @@ //! ([`SetupTrigger::Repair`]) and starts once the clone reads right. The //! repair moves the row to `Starting` for as long as it runs, so the session //! queue's own gate holds whatever else is queued. A repair that fails -//! leaves the row in `Error`, with its reason and a retry timer; one that is -//! refused, because switching would lose or move work or because another -//! live session is working in the clone, puts the row back to `Running` and -//! fails only the session that asked. +//! leaves the row in `Error`, with its reason and a retry timer. So does one +//! refused because switching would lose or move work, with no timer: the +//! error view's Retry is for whoever has dealt with the clone. A start that +//! finds another live session working in the clone asks for no repair and +//! leaves the row `Running`: only the session that asked fails, and the +//! session queue holds the branch's queued rows until that session is done. //! //! The read and the start it allows are one step as far as other starts and //! checkouts in the same clone go: the caller holds the clone's gate @@ -45,7 +47,7 @@ use tokio::sync::OwnedMutexGuard; use crate::branches; use crate::store::{self, Store}; -use super::clone::checkout_for; +use super::clone::{checkout_for, live_session_in_clone}; use super::guard::SetupGuard; use super::host::{blox_host, Checkout, SetupHost}; use super::{run_guarded, SetupRequest}; @@ -279,6 +281,20 @@ pub(crate) async fn verify_session_checkout( ); let repair = SetupRequest::repair_for(session_id); + // A session at work in the clone rules the repair out, and the database + // says so. The start fails here, with the cause, rather than take the + // row out of `Running` for a run that could only be blocked. A + // `disconnected` session is left to the repair, which has it settled + // first. + if let Some(live) = + live_session_in_clone(store, &checkout, &repair)?.filter(|live| !live.disconnected) + { + return Err(format!( + "Workspace '{workspace}' had {found} checked out, where this session's branch \ + '{expected}' was expected, and it could not be switched: {live}; try again once \ + that session has finished." + )); + } run_guarded(guard, store, host, &branch.id, repair) .await .map_err(|e| { diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index e13489053..8bc97595b 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -15,11 +15,14 @@ interface LocalSetupTask { /** Payload of the backend's `workspace-setup-error` event: why a setup run * left the branch in `error`, or null once it has no reason any more. * `userRequested` marks the failure of a setup the user asked for (a new - * branch, Retry, Resume), as opposed to one the backend ran on its own. */ + * branch, Retry, Resume), as opposed to one the backend ran on its own. + * `notice` comes with a setup that worked and did something the user + * should know of: it put commits it found on the base branch on this one. */ interface WorkspaceSetupErrorEvent { branchId: string; error: string | null; userRequested?: boolean; + notice?: string; } interface WorkspaceLifecycleHooks { @@ -350,6 +353,12 @@ class WorkspaceLifecycleController { this.setupErrorEventSeqByBranch.set(branchId, ++this.setupErrorEventSeq); if (error == null) { this.clearWorkspaceError(branchId); + if (event.notice) { + toast.warning('Workspace branch was given existing commits', { + description: event.notice, + duration: Infinity, + }); + } return; } this.setWorkspaceError(branchId, error); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 0d3a5deab..0d5db7f3d 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -12,6 +12,7 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { let refreshTransferredProject: Mock<(projectId: string) => Promise>; let eventListeners: Map void>>; let toastError: ReturnType; + let toastWarning: ReturnType; /** Deliver a backend event to the controller's listeners. */ function emit(event: string, payload: unknown) { @@ -60,7 +61,8 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { })); vi.doMock('svelte-sonner', () => { toastError = vi.fn(); - return { toast: { error: toastError } }; + toastWarning = vi.fn(); + return { toast: { error: toastError, warning: toastWarning } }; }); vi.doMock('../settings/preferences.svelte', () => ({ getPreferredAgent: () => 'claude', @@ -458,6 +460,33 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); + // A setup that put commits it found on the base branch on this branch + // worked, so there is no reason for the card. The user is still told, + // since the log is all that said so before. + it('tells the user when a setup brought commits along to the branch', async () => { + const lifecycle = await newController(); + await settle(); + emit('workspace-setup-error', { branchId: 'branch-1', error: 'could not fetch' }); + + const notice = "Branch 'feature' was created from the workstation's local 'main'"; + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + userRequested: false, + notice, + }); + expect(lifecycle.getWorkspaceErrors().has('branch-1')).toBe(false); + expect(toastWarning.mock.calls).toEqual([ + ['Workspace branch was given existing commits', { description: notice, duration: Infinity }], + ]); + expect(toastError).not.toHaveBeenCalled(); + + // A reason cleared with nothing to tell raises nothing. + emit('workspace-setup-error', { branchId: 'branch-1', error: null }); + expect(toastWarning).toHaveBeenCalledTimes(1); + lifecycle.stop(); + }); + // The backend reports every failed run, and a snapshot covers the ones // from before the window mounted. it('keeps setup reasons from the error event and the snapshot taken at start', async () => { From 9233d3e002349f9f5583c18c86abb823f996c5a3 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 19 Nov 2026 11:28:51 +1100 Subject: [PATCH 6/9] fix(staged): name the branch in the pull request pipeline and settle the last review items Applies the two warnings of the last review and the two small gaps it names. Nothing else from that review is taken up here. Pull request pipeline. The history and changed-files steps and every prompt an agent can be handed now name the local branch instead of HEAD, as the push's refspec already did, so the pipeline describes its own branch from a clone that is on a sibling branch or on the base. With that, the pipeline no longer reads what the clone has checked out and no longer asks for a repair. When the local branch does not exist, a new first step stops the pipeline before any step that can hand off to an agent: the session ends in error with the reason, no agent starts and nothing in the clone changes. An agent is therefore never handed a failed push to fix by creating the branch setup refused to create. The aborted pipeline now reports the step's words as the session error, and the pull request error shown to the user carries that reason. Queue hold. A queued row held behind a sibling branch's session now tells the user why, once per cause, as a notice on the existing workspace setup event with its own heading. On the first look and on each recheck a disconnected holder is handed to the reconcile service, so the hold does not wait out the reconciler's backoff. Transfer early exits. A transfer that is cancelled right after a resume, or whose second inspection fails, now writes Starting for the rows of the workspaces it resumed before returning, so setup picks them up. Sibling rule. Only sibling branch rows that have had a session count as possible owners of commits stranded on the base. A sibling that never ran one cannot have committed there and no longer makes the owner unclear. The test that pinned the stricter rule now gives the sibling a session, and new tests cover the sibling without one. No data model changes: one new store query, no new fields or types in the schema. Signed-off-by: Matt Toohey Co-Authored-By: Claude Fable 5.2 Signed-off-by: Matt Toohey --- .../src/project_transfer/orchestrator.rs | 53 +++- .../project_transfer/orchestrator_tests.rs | 73 ++++++ apps/staged/src-tauri/src/prs/context.rs | 6 +- apps/staged/src-tauri/src/prs/create_pr.rs | 74 ++++-- .../src-tauri/src/prs/tests/create_pr.rs | 229 +++++++++++++++++- apps/staged/src-tauri/src/prs/tests/push.rs | 2 +- apps/staged/src-tauri/src/session_commands.rs | 168 ++++++++++++- .../src-tauri/src/session_queue/retry.rs | 28 ++- apps/staged/src-tauri/src/session_runner.rs | 39 ++- apps/staged/src-tauri/src/store/sessions.rs | 13 + .../src-tauri/src/workspace_setup/checkout.rs | 12 +- .../src-tauri/src/workspace_setup/clone.rs | 24 +- .../src-tauri/src/workspace_setup/failures.rs | 36 ++- .../src-tauri/src/workspace_setup/host.rs | 4 +- .../src/workspace_setup/tests/checkout.rs | 111 ++++++++- .../src/workspace_setup/tests/wiring.rs | 79 ++++-- .../src-tauri/src/workspace_setup/verify.rs | 6 +- .../projects/workspaceLifecycle.svelte.ts | 7 +- .../projects/workspaceLifecycle.test.ts | 25 ++ .../listeners/sessionStatusListener.test.ts | 19 ++ .../lib/listeners/sessionStatusListener.ts | 11 +- 21 files changed, 916 insertions(+), 103 deletions(-) diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs index ed0d3f1e3..40fafbef2 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator.rs @@ -555,6 +555,38 @@ impl Run<'_> { Ok(()) } + /// The same for a transfer that stops between resuming a workspace and + /// reading its clones: a cancel that landed during the resume, or a + /// second inspection that failed. Nothing was read, so every row of a + /// workspace that came back says `Starting`, and setup is asked for it + /// once the transfer has ended. Best effort: the transfer is already + /// stopping for another reason, which is the one reported. + fn mark_resumed_rows_starting( + &self, + inspection: &Inspection, + resumed: &BTreeMap>, + ) { + for source in &inspection.branches { + let came_back = source + .workspace + .as_deref() + .is_some_and(|workspace| matches!(resumed.get(workspace), Some(None))); + if !came_back { + continue; + } + if let Err(e) = self + .store + .update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Starting) + { + log::warn!( + "[project_transfer] could not mark branch {} of a resumed workspace as \ + starting: {e}", + source.branch.id + ); + } + } + } + /// The preflight, again, now that the transfer is registered — with any /// suspended, stopped or failed source workspace resumed first, so its /// checkouts can be read. One that won't come back blocks the transfer. @@ -591,7 +623,16 @@ impl Run<'_> { .host .resume_workspace(&workspace, &|| self.guard.is_cancelled()) .err(); - self.guard_not_cancelled()?; + if let Err(cancelled) = self.guard_not_cancelled() { + // A resume that finished as the cancel landed left the + // workstation up with its rows still saying what it was + // resumed from, which nothing polls. + if error.is_none() { + resumed.insert(workspace, None); + } + self.mark_resumed_rows_starting(&inspection, &resumed); + return Err(cancelled.into()); + } let failed = error.is_some(); if let Some(e) = &error { log::warn!("[project_transfer] cannot resume {workspace}: {e}"); @@ -602,9 +643,15 @@ impl Run<'_> { break; } } - inspection = preflight::inspect_after_resuming( + inspection = match preflight::inspect_after_resuming( self.store, self.host, project_id, &resumed, &observer, - )?; + ) { + Ok(inspection) => inspection, + Err(e) => { + self.mark_resumed_rows_starting(&inspection, &resumed); + return Err(e.into()); + } + }; self.mark_resumed_rows_running(&inspection, &resumed)?; } let preflight = &inspection.preflight; diff --git a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs index 0a4e028c1..ecf5c0e44 100644 --- a/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs +++ b/apps/staged/src-tauri/src/project_transfer/orchestrator_tests.rs @@ -227,6 +227,8 @@ struct FakeHost { /// Called at the start of every script run, at either end, on the /// thread running it. script_hook: Mutex>>, + /// Called as a resume that worked returns, on the thread that asked. + resume_hook: Mutex>>, } impl FakeHost { @@ -249,6 +251,7 @@ impl FakeHost { script_vars: Mutex::default(), ignored_cap: Mutex::default(), script_hook: Mutex::default(), + resume_hook: Mutex::default(), }; for repo in repos { let origin = host.origin(repo); @@ -374,6 +377,11 @@ impl FakeHost { fn on_every_script(&self, hook: impl Fn(Side, &str) + Send + Sync + 'static) { *self.script_hook.lock().unwrap() = Some(Arc::new(hook)); } + + /// Run `hook` as every resume that worked returns, from now on. + fn on_resume(&self, hook: impl Fn() + Send + Sync + 'static) { + *self.resume_hook.lock().unwrap() = Some(Arc::new(hook)); + } } fn path(p: &Path) -> &str { @@ -518,6 +526,10 @@ impl TransferHost for FakeHost { return Err(TransferError::Cancelled.to_string()); } self.set_state(name, WorkspaceState::Running); + let hook = self.resume_hook.lock().unwrap().clone(); + if let Some(hook) = hook { + hook(); + } Ok(()) } @@ -2679,6 +2691,67 @@ fn a_resumed_row_whose_clone_is_on_another_branch_is_left_starting() { ); } +/// The two ways a transfer stops between a resume that worked and its +/// reading of the clones: a cancel that lands as the resume finishes, and a +/// second inspection that fails (here: cancelled while it reads). The +/// workstation is up and nothing was read, so the rows say `Starting`, which +/// the setup trigger takes once the transfer has ended. They used to stay +/// `Suspended`, which nothing polls. +#[test] +fn a_transfer_that_stops_right_after_a_resume_leaves_the_rows_starting() { + let f = Fixture::new(&[(WIDGETS, "feature")]); + f.transfer(true).result.expect("to remote"); + let suspend = || { + f.host.set_state(&f.workspace(), WorkspaceState::Suspended); + f.store + .update_workspace_status_by_workspace_name(&f.workspace(), &WorkspaceStatus::Suspended) + .unwrap(); + }; + let left_starting = |how: &str| { + let row = f.store.get_branch(&f.branches[0].id).unwrap().unwrap(); + assert_eq!( + row.workspace_status, + Some(WorkspaceStatus::Starting), + "{how}" + ); + assert!(!crate::project_transfer::is_branch_transferring(&row.id)); + assert_eq!( + crate::workspace_setup::startup_action(&row), + Some(crate::workspace_setup::StartupAction::SetUp), + "{how}" + ); + }; + let resumes = || { + let resume = format!("resume {}", f.workspace()); + f.host.calls().iter().filter(|c| **c == resume).count() + }; + + suspend(); + let project_id = f.project.id.clone(); + f.host.on_resume(move || { + registry::cancel(&project_id); + }); + let err = f.transfer(true).result.unwrap_err(); + assert!(err.contains("transfer was cancelled"), "{err}"); + assert_eq!(resumes(), 1); + left_starting("cancelled as the resume finished"); + + suspend(); + let resumed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = Arc::clone(&resumed); + f.host.on_resume(move || flag.store(true, Ordering::SeqCst)); + let project_id = f.project.id.clone(); + f.host.on_every_script(move |_, _| { + if resumed.load(Ordering::SeqCst) { + registry::cancel(&project_id); + } + }); + let err = f.transfer(true).result.unwrap_err(); + assert!(err.contains("transfer was cancelled"), "{err}"); + assert_eq!(resumes(), 2); + left_starting("the inspection after the resume failed"); +} + // --------------------------------------------------------------------------- // Panics // --------------------------------------------------------------------------- diff --git a/apps/staged/src-tauri/src/prs/context.rs b/apps/staged/src-tauri/src/prs/context.rs index f3438a4c7..3393662c8 100644 --- a/apps/staged/src-tauri/src/prs/context.rs +++ b/apps/staged/src-tauri/src/prs/context.rs @@ -106,9 +106,9 @@ pub(crate) fn resolve_branch_pipeline_context( /// to be on the branch, and put there when it is not. A rebase, squash, /// commit or pull in a clone that is on the base branch acts on the base. A /// push does not ask: it names its local branch in an explicit refspec and -/// works from any HEAD. The pull request pipeline asks, so the steps that -/// describe the branch look at it, and goes on whatever the answer. A no-op -/// for a local branch. +/// works from any HEAD. Neither does the pull request pipeline, whose steps +/// and prompts all name the local branch (`create_pr`). A no-op for a local +/// branch. /// /// Call it once the pipeline is known to run now, before its row goes /// `running`, and keep what it returns until it has diff --git a/apps/staged/src-tauri/src/prs/create_pr.rs b/apps/staged/src-tauri/src/prs/create_pr.rs index 9465f3a2b..0ec93559f 100644 --- a/apps/staged/src-tauri/src/prs/create_pr.rs +++ b/apps/staged/src-tauri/src/prs/create_pr.rs @@ -1,5 +1,17 @@ //! Creating a pull request: fetch the base, gather context, push, and hand //! off to an agent that writes the title and body and runs `gh pr create`. +//! +//! Nothing here depends on what the checkout has checked out. Two branches +//! of one repository share a workstation's clone, so it can be on a sibling +//! branch, with a session working there, or on the base branch. Every step +//! and every prompt names the local branch, as the push's refspec does, and +//! the pipeline makes no check of HEAD and asks for no checkout. +//! +//! What it does need is the local branch itself. A clone without it is one +//! branch setup has not put on the branch, possibly because it refused to +//! (`workspace_setup::checkout`): the first step stops the pipeline there, +//! with the reason as the session's error. No agent is started, so none is +//! handed a failed push to "fix" by creating the branch setup would not. use std::sync::Arc; @@ -7,14 +19,29 @@ use crate::git; use crate::session_runner; use crate::store::{FailureStrategy, PipelineStep, Store}; -use super::context::{base_branch_name, ensure_pipeline_checkout, resolve_branch_pipeline_context}; +use super::context::{base_branch_name, resolve_branch_pipeline_context}; use super::git_commands::{git_fetch_with_fallback, git_push_with_fallback, push_refspec}; use super::pipeline::start_pipeline_for_branch; +/// The label of the step that stops the pipeline when the checkout has no +/// local branch to push. +pub(super) const CHECK_LOCAL_BRANCH_LABEL: &str = "Check local branch"; + +/// Exits 0 when the checkout has `local_branch`. When it does not, says so +/// for the user and exits 1. Anything else (no repository, a command that +/// never ran) keeps git's own exit status and words, so it is not reported +/// as a missing branch. +fn check_local_branch_command(local_branch: &str) -> String { + format!( + r#"if git show-ref --verify --quiet refs/heads/{local_branch}; then :; else rc=$?; if [ "$rc" -eq 1 ]; then echo "Local branch '{local_branch}' does not exist in this clone, so there is nothing to push and no pull request was created. Branch setup creates the branch: if the branch shows a setup error, deal with what it reports and retry setup, then create the pull request again." >&2; fi; exit "$rc"; fi"# + ) +} + fn create_pr_handoff_prompt( pr_type: &str, base_branch: &str, base_ref: &str, + local_branch: &str, head_branch: &str, draft_flag: &str, opening: &str, @@ -28,13 +55,14 @@ fn create_pr_handoff_prompt( Use the context below as a starting point, and inspect the branch changes as needed. Requirements: +- The {pr_type} is for the local branch `{local_branch}`. The checkout may have another branch checked out, with another session working on it: name `{local_branch}` explicitly instead of relying on HEAD, and do not check out, switch, create or reset branches. If `{local_branch}` does not exist locally, stop and say so instead of creating it. - Treat `{base_ref}` as the comparison base. The local `{base_branch}` branch may be stale; do not use it for diff or log comparisons. -- If you inspect branch changes, compare `$(git merge-base {base_ref} HEAD)..HEAD`. +- If you inspect branch changes, compare `$(git merge-base {base_ref} {local_branch})..{local_branch}`. - Choose a conventional-commit-style PR title, using the most appropriate type (feat, fix, refactor, docs, style, test, chore, perf, ci, or build) based on the actual changes. - Write a concise PR body that summarizes the changes. - Run `gh pr create --base {base_branch} --head {head_branch} --title --body <body>{draft_flag}` to create the {pr_type}. `{head_branch}` is the branch on origin the previous step pushed to; it may differ from the local branch name. - Do not use `--fill-first`. -- If `gh pr create` reports that a PR already exists, run `gh pr view --json url --jq .url`, update the title/body with `gh pr edit` if needed, and continue. +- If `gh pr create` reports that a PR already exists, run `gh pr view {head_branch} --json url --jq .url`, update the title/body with `gh pr edit {head_branch}` if needed, and continue. - When done, output exactly `PR_URL: <url>`. Context from prior steps: @@ -55,6 +83,13 @@ pub(super) fn build_create_pr_pipeline_steps( let refspec = push_refspec(local_branch, remote_branch); vec![ + // First, before any step that can hand off to an agent: with no + // local branch the pipeline ends here (see the module docs). + PipelineStep::Command { + label: CHECK_LOCAL_BRANCH_LABEL.to_string(), + command: check_local_branch_command(local_branch), + on_failure: FailureStrategy::Abort { marker: None }, + }, PipelineStep::Command { label: "Fetch latest base".to_string(), command: git_fetch_with_fallback(base_branch), @@ -63,10 +98,11 @@ pub(super) fn build_create_pr_pipeline_steps( pr_type, base_branch, &base_ref, + local_branch, remote_branch, draft_flag, &format!( - "The fetch failed while creating a {pr_type}. Diagnose and fix the issue if needed, then inspect this branch against `{base_ref}`, push the branch, and create or recover the {pr_type}." + "The fetch failed while creating a {pr_type}. Diagnose and fix the issue if needed, then inspect `{local_branch}` against `{base_ref}`, push it with exactly `git push -u origin {refspec}`, and create or recover the {pr_type}." ), ), }, @@ -74,14 +110,14 @@ pub(super) fn build_create_pr_pipeline_steps( PipelineStep::Command { label: "View commit history".to_string(), command: format!( - r#"base_commit=$(git merge-base {base_ref} HEAD) && git log --oneline "$base_commit"..HEAD"# + r#"base_commit=$(git merge-base {base_ref} {local_branch}) && git log --oneline "$base_commit"..{local_branch}"# ), on_failure: FailureStrategy::Continue, }, PipelineStep::Command { label: "View changed files".to_string(), command: format!( - r#"base_commit=$(git merge-base {base_ref} HEAD) && git diff "$base_commit"..HEAD --stat"# + r#"base_commit=$(git merge-base {base_ref} {local_branch}) && git diff "$base_commit"..{local_branch} --stat"# ), on_failure: FailureStrategy::Continue, }, @@ -93,6 +129,7 @@ pub(super) fn build_create_pr_pipeline_steps( pr_type, base_branch, &base_ref, + local_branch, remote_branch, draft_flag, &format!( @@ -107,9 +144,10 @@ pub(super) fn build_create_pr_pipeline_steps( pr_type, base_branch, &base_ref, + local_branch, remote_branch, draft_flag, - &format!("Create a {pr_type} for the current branch."), + &format!("Create a {pr_type} for the local branch `{local_branch}`."), ), }, ] @@ -126,20 +164,9 @@ pub(crate) async fn start_create_pr_pipeline_for_branch( ) -> Result<String, String> { crate::project_transfer::ensure_branch_not_transferring(&branch_id)?; let ctx = resolve_branch_pipeline_context(&store, &branch_id)?; - // The clone is read and, when it can be, put on the branch, so the - // steps that describe the branch look at it. The pipeline does not fail - // on that: its push names the local branch in an explicit refspec and - // works from any HEAD, a sibling branch with a live session included. - // Held until this returns, by when the pipeline's row is `running`. - let _checkout = ensure_pipeline_checkout(&app_handle, &store, &ctx.branch, None) - .await - .unwrap_or_else(|e| { - log::warn!( - "[create_pr] branch {branch_id}: creating the pull request from a clone that \ - is not on the branch: {e}" - ); - crate::workspace_setup::verify::CheckoutHold::none() - }); + // No read of what the clone has checked out and no repair: every step + // names the local branch, so the pipeline runs the same from any HEAD + // (see the module docs). let base_branch = base_branch_name(&ctx.branch); let is_draft = draft.unwrap_or(false); @@ -150,7 +177,10 @@ pub(crate) async fn start_create_pr_pipeline_for_branch( "pull request" }; - // Build the pipeline steps for PR creation. + // Build the pipeline steps for PR creation. The session's own prompt + // below is its label and what a resume reads its type from + // (`infer_branch_resume_session_type`); the prompts an agent is handed + // are in the steps and name the branch. let remote_branch = crate::branches::resolve_remote_branch(&store, &ctx.branch).await; let steps = build_create_pr_pipeline_steps( pr_type, diff --git a/apps/staged/src-tauri/src/prs/tests/create_pr.rs b/apps/staged/src-tauri/src/prs/tests/create_pr.rs index 902883b01..8166d9cf3 100644 --- a/apps/staged/src-tauri/src/prs/tests/create_pr.rs +++ b/apps/staged/src-tauri/src/prs/tests/create_pr.rs @@ -1,6 +1,8 @@ //! The create-PR pipeline steps. use super::*; +use crate::prs::create_pr::CHECK_LOCAL_BRANCH_LABEL; +use crate::test_utils::{run_git_in, TempDir, TempGitRepo}; #[test] fn create_pr_pipeline_fetches_base_and_uses_origin_merge_base_for_context() { @@ -12,40 +14,249 @@ fn create_pr_pipeline_fetches_base_and_uses_origin_merge_base_for_context() { "feature-branch", ); - assert_eq!(steps.len(), 5); + assert_eq!(steps.len(), 6); let (label, command, on_failure) = command_at(&steps, 0); + assert_eq!(label, CHECK_LOCAL_BRANCH_LABEL); + assert!( + command.starts_with("if git show-ref --verify --quiet refs/heads/feature-branch-2; then"), + "{command}" + ); + assert!(matches!( + on_failure, + FailureStrategy::Abort { marker: None } + )); + + let (label, command, on_failure) = command_at(&steps, 1); assert_eq!(label, "Fetch latest base"); assert_eq!( command, "if ! git fetch origin main; then git -c 'url.https://github.com/.insteadOf=git@github.com:' fetch origin main; fi" ); - assert!(matches!(on_failure, FailureStrategy::HandoffToAi { .. })); + let FailureStrategy::HandoffToAi { prompt_template } = on_failure else { + panic!("the fetch hands off: {on_failure:?}"); + }; + assert!(prompt_template.contains( + "inspect `feature-branch-2` against `origin/main`, push it with exactly \ + `git push -u origin feature-branch-2:feature-branch`" + )); - let (_, command, _) = command_at(&steps, 1); + let (_, command, _) = command_at(&steps, 2); assert_eq!( command, - r#"base_commit=$(git merge-base origin/main HEAD) && git log --oneline "$base_commit"..HEAD"# + r#"base_commit=$(git merge-base origin/main feature-branch-2) && git log --oneline "$base_commit"..feature-branch-2"# ); assert!(!command.contains("git log --oneline main")); - let (_, command, _) = command_at(&steps, 2); + let (_, command, _) = command_at(&steps, 3); assert_eq!( command, - r#"base_commit=$(git merge-base origin/main HEAD) && git diff "$base_commit"..HEAD --stat"# + r#"base_commit=$(git merge-base origin/main feature-branch-2) && git diff "$base_commit"..feature-branch-2 --stat"# ); assert!(!command.contains("git diff main")); - let (label, command, _) = command_at(&steps, 3); + let (label, command, _) = command_at(&steps, 4); assert_eq!(label, "Push to remote"); assert_eq!( command, "if ! git push -u origin feature-branch-2:feature-branch; then git -c 'url.https://github.com/.insteadOf=git@github.com:' push -u origin feature-branch-2:feature-branch; fi" ); - let (label, prompt) = ai_prompt_at(&steps, 4); + let (label, prompt) = ai_prompt_at(&steps, 5); assert_eq!(label, "Create PR"); + assert!(prompt.contains("Create a pull request for the local branch `feature-branch-2`.")); assert!(prompt.contains("local `main` branch may be stale")); - assert!(prompt.contains("$(git merge-base origin/main HEAD)..HEAD")); + assert!(prompt.contains("$(git merge-base origin/main feature-branch-2)..feature-branch-2")); assert!(prompt.contains("gh pr create --base main --head feature-branch ")); + assert!(prompt.contains("`gh pr view feature-branch --json url --jq .url`")); + assert!(prompt.contains("`gh pr edit feature-branch`")); +} + +/// Every command a step runs and every prompt an agent could be handed. +fn commands_and_prompts(steps: &[PipelineStep]) -> Vec<&str> { + let mut texts = Vec::new(); + for step in steps { + match step { + PipelineStep::Command { + command, + on_failure, + .. + } => { + texts.push(command.as_str()); + if let FailureStrategy::HandoffToAi { prompt_template } = on_failure { + texts.push(prompt_template.as_str()); + } + } + PipelineStep::AiHandoff { + prompt_template, .. + } => texts.push(prompt_template.as_str()), + } + } + texts +} + +/// Run a step's command in `dir` the way the pipeline does, with `sh` and +/// with `dash` where the machine has it (a workstation's `/bin/sh`), and +/// return its exit code and what it printed. +fn run_step(dir: &std::path::Path, command: &str) -> Vec<(Option<i32>, String)> { + let mut shells = vec!["sh"]; + if std::path::Path::new("/bin/dash").exists() { + shells.push("/bin/dash"); + } + shells + .into_iter() + .map(|shell| { + let mut sh = std::process::Command::new(shell); + sh.arg("-c").arg(command).current_dir(dir); + crate::git::strip_git_env(&mut sh); + let output = sh.output().unwrap(); + let printed = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + (output.status.code(), printed) + }) + .collect() +} + +/// A clone of a bare origin, with `feature-b` one commit ahead of +/// `origin/main` and not pushed. +fn clone_with_feature_b() -> (TempDir, TempDir) { + let seed = TempGitRepo::new(); + seed.write_file("file.txt", "base\n"); + seed.commit("base"); + let origin = TempDir::new("origin"); + let origin_path = origin.path().to_str().unwrap(); + run_git_in(seed.path(), &["clone", "--bare", ".", origin_path]); + let clone = TempDir::new("workstation"); + run_git_in( + origin.path(), + &["clone", origin_path, clone.path().to_str().unwrap()], + ); + run_git_in(clone.path(), &["config", "user.email", "test@example.com"]); + run_git_in(clone.path(), &["config", "user.name", "Test"]); + run_git_in(clone.path(), &["checkout", "-b", "feature-b"]); + std::fs::write(clone.path().join("b.txt"), "b\n").unwrap(); + run_git_in(clone.path(), &["add", "b.txt"]); + run_git_in(clone.path(), &["commit", "-m", "feature-b's work"]); + (origin, clone) +} + +/// The pipeline describes its own branch whatever the clone has checked +/// out. Here that is a sibling branch with work of its own, as it is while +/// a sibling's session has a shared clone: the history and changed-files +/// steps used to read HEAD, and the agent was told to compare against HEAD, +/// so the pull request for `feature-b` was written from `feature-a`'s +/// commits. No command and no prompt reads HEAD now, and the prompt tells +/// the agent to leave the checkout where it is. +#[test] +fn the_pull_request_describes_its_branch_from_a_clone_on_a_sibling_branch() { + let (_origin, clone) = clone_with_feature_b(); + run_git_in( + clone.path(), + &["checkout", "-b", "feature-a", "origin/main"], + ); + std::fs::write(clone.path().join("a.txt"), "a\n").unwrap(); + run_git_in(clone.path(), &["add", "a.txt"]); + run_git_in(clone.path(), &["commit", "-m", "feature-a's work"]); + + let steps = + build_create_pr_pipeline_steps("pull request", "main", "", "feature-b", "feature-b"); + for text in commands_and_prompts(&steps) { + assert!( + !text.contains("HEAD)") && !text.contains("..HEAD"), + "{text}" + ); + assert!(!text.contains("the current branch"), "{text}"); + } + let (_, prompt) = ai_prompt_at(&steps, 5); + assert!( + prompt.contains("do not check out, switch, create or reset branches"), + "{prompt}" + ); + + for (code, printed) in run_step(clone.path(), command_at(&steps, 0).1) { + assert_eq!(code, Some(0), "{printed}"); + } + for (code, history) in run_step(clone.path(), command_at(&steps, 2).1) { + assert_eq!(code, Some(0), "{history}"); + assert!(history.contains("feature-b's work"), "{history}"); + assert!(!history.contains("feature-a's work"), "{history}"); + } + for (code, changed) in run_step(clone.path(), command_at(&steps, 3).1) { + assert_eq!(code, Some(0), "{changed}"); + assert!(changed.contains("b.txt"), "{changed}"); + assert!(!changed.contains("a.txt"), "{changed}"); + } + assert_eq!( + run_git_in(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), + "feature-a" + ); +} + +/// The state a refused or blocked setup leaves: the clone on `main` with +/// the agent's commits and no local branch. The push would fail there and +/// hand an agent "fix it, then retry the push", which it does by creating +/// the branch setup would not. The first step stops the pipeline instead, +/// before any step that hands off, with words for the user and nothing +/// changed. A check that could not run (no repository) is not reported as a +/// missing branch, and stops the pipeline all the same. +#[test] +fn a_clone_without_the_local_branch_stops_the_pull_request_before_any_agent() { + let (_origin, clone) = clone_with_feature_b(); + run_git_in(clone.path(), &["checkout", "main"]); + std::fs::write(clone.path().join("stranded.txt"), "work\n").unwrap(); + run_git_in(clone.path(), &["add", "stranded.txt"]); + run_git_in(clone.path(), &["commit", "-m", "agent work"]); + let before = run_git_in( + clone.path(), + &["for-each-ref", "--format=%(refname) %(objectname)"], + ); + + let steps = + build_create_pr_pipeline_steps("pull request", "main", "", "feature-new", "feature-new"); + let hands_off = |step: &PipelineStep| match step { + PipelineStep::Command { on_failure, .. } => { + matches!(on_failure, FailureStrategy::HandoffToAi { .. }) + } + PipelineStep::AiHandoff { .. } => true, + }; + assert_eq!(steps.iter().position(hands_off), Some(1)); + let (label, command, on_failure) = command_at(&steps, 0); + assert_eq!(label, CHECK_LOCAL_BRANCH_LABEL); + assert!(matches!( + on_failure, + FailureStrategy::Abort { marker: None } + )); + + for (code, printed) in run_step(clone.path(), command) { + assert_eq!(code, Some(1), "{printed}"); + assert!( + printed.starts_with("Local branch 'feature-new' does not exist in this clone"), + "{printed}" + ); + assert!(printed.contains("no pull request was created"), "{printed}"); + assert!(printed.contains("retry setup"), "{printed}"); + } + assert_eq!( + run_git_in( + clone.path(), + &["for-each-ref", "--format=%(refname) %(objectname)"] + ), + before + ); + assert_eq!( + run_git_in(clone.path(), &["rev-parse", "--abbrev-ref", "HEAD"]), + "main" + ); + + let no_repository = TempDir::new("no-repository"); + for (code, printed) in run_step(no_repository.path(), command) { + assert!(code != Some(0) && code != Some(1), "{code:?}: {printed}"); + assert!( + !printed.contains("does not exist in this clone"), + "{printed}" + ); + } } diff --git a/apps/staged/src-tauri/src/prs/tests/push.rs b/apps/staged/src-tauri/src/prs/tests/push.rs index 98934ac05..0bc942867 100644 --- a/apps/staged/src-tauri/src/prs/tests/push.rs +++ b/apps/staged/src-tauri/src/prs/tests/push.rs @@ -318,7 +318,7 @@ fn a_push_works_from_a_clone_that_has_a_sibling_branch_checked_out() { let push = build_push_pipeline_steps("feature-b", "feature-b", PUSH); let pr = build_create_pr_pipeline_steps("pull request", "main", "", "feature-b", "feature-b"); let (_, command, _) = command_at(&push, 0); - assert_eq!(command_at(&pr, 3).1, command); + assert_eq!(command_at(&pr, 4).1, command); let mut sh = std::process::Command::new("sh"); sh.arg("-c").arg(command).current_dir(clone.path()); crate::git::strip_git_env(&mut sh); diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index 8141c2f12..29628bf42 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -4525,23 +4525,64 @@ fn retry_or_fail_queued_start_before_claim<R: tauri::Runtime>( /// row's failure and clears on its own, so the row keeps its place and its /// attempts, the rows behind it wait with it, and the branch is looked at /// again shortly: the queue moves once the other session is done. -fn hold_queue_behind_sibling_session( +/// +/// A held row stays `queued` with no error, so the user is told why it +/// waits, once per cause, as a notice on the branch's setup event +/// (`workspace_setup::failures::notify_queue_held`). And a `disconnected` +/// holder is handed to the reconcile service on every look, this one and +/// each recheck: left to the service's own backoff it could hold the queue +/// for half an hour after its turn was over. +fn hold_queue_behind_sibling_session<R: tauri::Runtime>( store: &Store, + app_handle: &tauri::AppHandle<R>, kicker: &Arc<crate::session_queue::BranchQueueKicker>, branch: &store::Branch, session_id: &str, ) -> bool { + let Some(hold) = sibling_session_hold(store, kicker, branch, session_id) else { + return false; + }; + if let Some(notice) = &hold.announce { + crate::workspace_setup::failures::notify_queue_held(app_handle, &branch.id, notice); + } + if let Some(holder) = hold.disconnected_holder { + if let Err(e) = crate::session_runner::reconcile::reconcile_session(holder.clone()) { + log::debug!("[session_queue] could not ask for a check of session {holder}: {e}"); + } + } + true +} + +/// What a hold behind a sibling branch's session leaves its caller to do. +#[derive(Debug, PartialEq, Eq)] +struct SiblingHold { + /// Why the row waits, when the user has not been told this yet. + announce: Option<String>, + /// The session holding the clone, when it is `disconnected`: the + /// reconcile service is to look at it now. + disconnected_holder: Option<String>, +} + +/// [`hold_queue_behind_sibling_session`] up to what it tells the user and +/// the reconcile service: the look at the database, the hold and the kick +/// that rechecks it. `None` when no sibling's session has the clone. +fn sibling_session_hold( + store: &Store, + kicker: &Arc<crate::session_queue::BranchQueueKicker>, + branch: &store::Branch, + session_id: &str, +) -> Option<SiblingHold> { use crate::session_queue::retry::QUEUE_HOLD_RECHECK; let live = match crate::workspace_setup::clone::sibling_session_in_clone(store, branch) { Ok(Some(live)) => live, - Ok(None) => return false, + Ok(None) => return None, Err(e) => { log::warn!( "[session_queue] could not look for sessions sharing the clone of branch {}: {e}", branch.id ); - return false; + return None; } }; log::info!( @@ -4549,17 +4590,26 @@ fn hold_queue_behind_sibling_session( it: {live}", branch.id ); - kicker.start_attempts().hold( + let reason = format!( + "A queued session of branch '{}' is waiting: {live}. It starts once that session has \ + finished.", + branch.branch_name + ); + let news = kicker.start_attempts().hold( &branch.id, session_id, std::time::Instant::now() + QUEUE_HOLD_RECHECK, + &reason, ); kicker.kick_after( &branch.id, QUEUE_HOLD_RECHECK, crate::session_queue::KickSource::StartRetry, ); - true + Some(SiblingHold { + announce: news.then_some(reason), + disconnected_holder: live.disconnected.then_some(live.session_id), + }) } fn note_session_schedule() -> BranchSessionSchedule { @@ -6460,7 +6510,13 @@ pub(crate) async fn drain_branch_queue( // while that session lasts: asked of the database, not the clone. if kicker.as_ref().is_some_and(|kicker| { kicker.start_attempts().is_held(&branch_id, &session.id) - && hold_queue_behind_sibling_session(&store, kicker, &branch, &session.id) + && hold_queue_behind_sibling_session( + &store, + &app_handle, + kicker, + &branch, + &session.id, + ) }) { break; } @@ -6559,7 +6615,13 @@ pub(crate) async fn drain_branch_queue( break; } if kicker.as_ref().is_some_and(|kicker| { - hold_queue_behind_sibling_session(&store, kicker, &branch, &session_id) + hold_queue_behind_sibling_session( + &store, + &app_handle, + kicker, + &branch, + &session_id, + ) }) { break; } @@ -12614,7 +12676,7 @@ mod tests { store.create_session(&first).unwrap(); let second = store::Session::new_queued("note").with_branch(&branch.id); store.create_session(&second).unwrap(); - let hold = || hold_queue_behind_sibling_session(&store, &kicker, &branch, &first.id); + let hold = || sibling_session_hold(&store, &kicker, &branch, &first.id).is_some(); // Nobody else in the clone: whatever failed is the row's own. assert!(!hold()); @@ -12662,6 +12724,96 @@ mod tests { assert!(!attempts.is_held(&branch.id, &first.id)); } + /// A held row stays `queued` with no error, and its reason used to go to + /// the log alone. The user is told why it waits as a notice on the + /// branch's setup event, once per cause and not on every recheck. A + /// holder that is `disconnected` is named for the reconcile service on + /// every look, so the hold does not wait out that service's backoff. + #[test] + fn a_held_queue_says_why_once_and_has_a_disconnected_holder_checked_on_every_look() { + use crate::session_queue::BranchQueueKicker; + use tauri::Listener; + + let (store, branch) = setup_remote_branch_store(store::WorkspaceStatus::Running); + let mut sibling = + store::Branch::new_remote(&branch.project_id, "sibling", "main", "test-workspace"); + sibling.workspace_status = Some(store::WorkspaceStatus::Running); + store.create_branch(&sibling).unwrap(); + let kicker = Arc::new(BranchQueueKicker::recording()); + let first = store::Session::new_queued("note").with_branch(&branch.id); + store.create_session(&first).unwrap(); + let agent = store::Session::new_running("work", Path::new("/tmp")).with_branch(&sibling.id); + store.create_session(&agent).unwrap(); + let app = mock_app(); + let events = Arc::new(std::sync::Mutex::new(Vec::<String>::new())); + let seen = Arc::clone(&events); + app.handle() + .listen_any(crate::workspace_setup::failures::EVENT, move |event| { + seen.lock().unwrap().push(event.payload().to_string()); + }); + + // The first look tells every client; the rechecks say nothing new. + for _ in 0..3 { + assert!(hold_queue_behind_sibling_session( + &store, + app.handle(), + &kicker, + &branch, + &first.id + )); + } + let told = events.lock().unwrap().clone(); + assert_eq!(told.len(), 1, "{told:?}"); + let event: serde_json::Value = serde_json::from_str(&told[0]).unwrap(); + assert_eq!(event["branchId"], branch.id.as_str()); + assert_eq!(event["error"], serde_json::Value::Null); + assert_eq!(event["noticeTitle"], "Queued session is waiting"); + let notice = event["notice"].as_str().unwrap(); + for part in [ + branch.branch_name.as_str(), + "a session of branch 'sibling' is running in the same clone", + "starts once that session has finished", + ] { + assert!(notice.contains(part), "{part}: {notice}"); + } + let row = store.get_session(&first.id).unwrap().unwrap(); + assert_eq!(row.status, store::SessionStatus::Queued); + assert_eq!(row.error_message, None); + + // The holder loses its connection: a new cause, told once, and the + // session is named for the reconcile service on each look. + store + .update_session_status(&agent.id, store::SessionStatus::Disconnected, None, None) + .unwrap(); + let look = || sibling_session_hold(&store, &kicker, &branch, &first.id).unwrap(); + let lost = look(); + assert!( + lost.announce + .as_deref() + .is_some_and(|notice| notice.contains("lost its connection")), + "{lost:?}" + ); + assert_eq!(lost.disconnected_holder.as_deref(), Some(agent.id.as_str())); + for _ in 0..3 { + assert_eq!( + look(), + SiblingHold { + announce: None, + disconnected_holder: Some(agent.id.clone()), + } + ); + } + + // Settled: nothing holds the row, and nobody is asked about. + store + .update_session_status(&agent.id, store::SessionStatus::Completed, None, None) + .unwrap(); + assert_eq!( + sibling_session_hold(&store, &kicker, &branch, &first.id), + None + ); + } + /// A pre-claim start failure is retried before it is recorded: the first /// failures leave the row `queued`, in backoff, with a delayed kick /// scheduled for the branch — nothing else would re-kick it — and the diff --git a/apps/staged/src-tauri/src/session_queue/retry.rs b/apps/staged/src-tauri/src/session_queue/retry.rs index 0b04a27b4..8df670998 100644 --- a/apps/staged/src-tauri/src/session_queue/retry.rs +++ b/apps/staged/src-tauri/src/session_queue/retry.rs @@ -68,8 +68,9 @@ struct Attempt { /// When the next start may be tried; a kick before then leaves the row /// where it is rather than spending an attempt on it. retry_at: Instant, - /// The row is waiting on something other than its own failed start. - held: bool, + /// The row is waiting on something other than its own failed start: + /// what the user was last told it waits on. + held: Option<String>, } /// Per-row pre-claim start failures, by branch. @@ -98,7 +99,7 @@ impl QueuedStartAttempts { Attempt { failures, retry_at: now + delay, - held: false, + held: None, }, ); } @@ -113,19 +114,30 @@ impl QueuedStartAttempts { } /// Keep `session_id` where it is until `until` without counting a failed - /// start against it. - pub(crate) fn hold(&self, branch_id: &str, session_id: &str, until: Instant) { + /// start against it. `reason` is what it waits on; returns whether that + /// is news: the row was not held, or was held over something else. A + /// hold is looked at again every few seconds, and the user is told once. + pub(crate) fn hold( + &self, + branch_id: &str, + session_id: &str, + until: Instant, + reason: &str, + ) -> bool { let mut by_branch = self.by_branch.lock().unwrap(); let rows = by_branch.entry(branch_id.to_string()).or_default(); - let failures = rows.get(session_id).map_or(0, |a| a.failures); + let previous = rows.get(session_id); + let failures = previous.map_or(0, |a| a.failures); + let news = previous.and_then(|a| a.held.as_deref()) != Some(reason); rows.insert( session_id.to_string(), Attempt { failures, retry_at: until, - held: true, + held: Some(reason.to_string()), }, ); + news } /// Whether `session_id` was last left where it is by [`Self::hold`]. @@ -135,7 +147,7 @@ impl QueuedStartAttempts { .unwrap() .get(branch_id) .and_then(|rows| rows.get(session_id)) - .is_some_and(|attempt| attempt.held) + .is_some_and(|attempt| attempt.held.is_some()) } /// Whether `session_id` is waiting out the delay from a failed start. diff --git a/apps/staged/src-tauri/src/session_runner.rs b/apps/staged/src-tauri/src/session_runner.rs index 3473417f3..0136e4214 100644 --- a/apps/staged/src-tauri/src/session_runner.rs +++ b/apps/staged/src-tauri/src/session_runner.rs @@ -5422,11 +5422,13 @@ fn mark_ai_handoff_step_failed<R: tauri::Runtime>( /// Error message for an aborted pipeline, or `None` when the abort is an expected /// outcome the session should still complete with. /// -/// Only a pull produces an error. A push rejected as non-fast-forward completes so +/// Only a push produces none. A push rejected as non-fast-forward completes so /// the frontend can read the steps and offer a force push, but a pull that can't /// fast-forward is a dead end whose fix is a user decision (rebase onto origin, or /// reset to origin) — and a pull drained off the branch queue runs headless, so -/// the error status is the only way the user hears about it. +/// the error status is the only way the user hears about it. The pull request +/// pipeline's abort is a dead end too: the checkout has no local branch to push +/// (`prs::create_pr`), and a completed session would read as "no PR URL found". /// /// The failing step's output is read back from the persisted pipeline, which /// `run_pipeline` writes before it returns `Aborted`. @@ -5435,7 +5437,7 @@ fn aborted_pipeline_error( store: &Store, step_index: usize, ) -> Option<String> { - if config.pipeline.kind.as_ref() != Some(&PipelineKind::Pull) { + if config.pipeline.kind.as_ref() == Some(&PipelineKind::Push) { return None; } @@ -5446,7 +5448,11 @@ fn aborted_pipeline_error( .and_then(|session| session.pipeline) .and_then(|pipeline| pipeline.steps.into_iter().nth(step_index)); let Some(step) = step else { - return Some("Pull failed".to_string()); + let what = match config.pipeline.kind { + Some(PipelineKind::Pull) => "Pull", + _ => "Pipeline", + }; + return Some(format!("{what} failed")); }; let detail = step @@ -12788,13 +12794,14 @@ draft file" /// A session whose pipeline recorded `step_index` as failed, mirroring what /// `run_pipeline` persists before it returns `Aborted`. fn store_with_aborted_pipeline( - kind: PipelineKind, + kind: impl Into<Option<PipelineKind>>, steps: &[PipelineStep], step_index: usize, output: Option<&str>, ) -> (Store, PipelineConfig) { let store = Store::in_memory().unwrap(); - let mut pipeline = PipelineExecution::from_steps(steps).with_kind(kind.clone()); + let mut pipeline = PipelineExecution::from_steps(steps); + pipeline.kind = kind.into(); pipeline.steps[step_index].status = StepStatus::Failed; pipeline.steps[step_index].output = output.map(str::to_string); let mut session = @@ -12858,6 +12865,26 @@ draft file" ); } + /// The pull request pipeline has no kind and aborts in one place: its + /// first step found no local branch to push. That ends the session in + /// `error` with the step's words, not `completed`, which the frontend + /// would report as a pull request session that printed no URL. + #[test] + fn aborted_pull_request_pipeline_reports_why_it_stopped() { + let steps = vec![abort_step("Check local branch")]; + let (store, config) = store_with_aborted_pipeline( + None::<PipelineKind>, + &steps, + 0, + Some("Local branch 'feature' does not exist in this clone.\n"), + ); + + assert_eq!( + aborted_pipeline_error(&config, &store, 0).unwrap(), + "Check local branch failed:\n\nLocal branch 'feature' does not exist in this clone." + ); + } + #[test] fn aborted_push_completes_without_an_error_message() { // A push rejected as non-fast-forward is an expected outcome: the session diff --git a/apps/staged/src-tauri/src/store/sessions.rs b/apps/staged/src-tauri/src/store/sessions.rs index 52b3fd4df..987ca6dd3 100644 --- a/apps/staged/src-tauri/src/store/sessions.rs +++ b/apps/staged/src-tauri/src/store/sessions.rs @@ -1230,6 +1230,19 @@ impl Store { self.get_sessions_for_branch_where(branch_id, "status IN ('running', 'disconnected')") } + /// Whether a session of the branch has ever left `queued`: one is + /// running, or has run. Only such a branch can have had an agent commit + /// in its checkout. + pub fn has_started_session_for_branch(&self, branch_id: &str) -> Result<bool, StoreError> { + let conn = self.conn.lock().unwrap(); + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM sessions WHERE status != 'queued' AND branch_id = ?1", + params![branch_id], + |row| row.get(0), + )?; + Ok(count > 0) + } + fn get_sessions_for_branch_where( &self, branch_id: &str, diff --git a/apps/staged/src-tauri/src/workspace_setup/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/checkout.rs index 141fec667..d5ce31055 100644 --- a/apps/staged/src-tauri/src/workspace_setup/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/checkout.rs @@ -93,8 +93,10 @@ pub(crate) struct BranchCheckout<'a> { /// behind on the base by mistake, and they are neither brought along /// nor a reason to refuse. pub(crate) base_is_worked_on: bool, - /// The other branches Staged works on in this clone. Commits stranded - /// on the base could be any of theirs that the clone doesn't have yet. + /// The other branches Staged works on in this clone that have had a + /// session. Commits stranded on the base could be any of theirs that + /// the clone doesn't have yet. A branch that never ran a session made + /// no commits and is left out. pub(crate) siblings: &'a [String], } @@ -388,9 +390,11 @@ fn stranded_range(exclude_base: &str) -> [&str; 5] { /// The reflog is the evidence because ancestry is not: a base that was /// force-pushed and one that simply moved on both leave local base /// diverged from origin's. -/// - No other branch Staged works on in this clone is missing from it. A -/// branch the clone doesn't have was never checked out here, so its +/// - No other branch that has had a session in this clone is missing from +/// it. A branch the clone doesn't have was never checked out here, so its /// sessions committed on the base as well, and the commits could be its. +/// A branch that never had a session committed nowhere and is not asked +/// about ([`BranchCheckout::siblings`]). fn refuse_unless_adoptable( git: &GitRunner<'_>, checkout: &BranchCheckout<'_>, diff --git a/apps/staged/src-tauri/src/workspace_setup/clone.rs b/apps/staged/src-tauri/src/workspace_setup/clone.rs index 5d9b26384..d9034b694 100644 --- a/apps/staged/src-tauri/src/workspace_setup/clone.rs +++ b/apps/staged/src-tauri/src/workspace_setup/clone.rs @@ -29,12 +29,25 @@ pub(crate) fn checkout_for(store: &Arc<Store>, branch: &store::Branch) -> Result .ok_or("Branch has no workspace name")?; let clone_dir = branches::resolve_branch_clone_dir(store, branch)?; let base_ref = branches::normalize_branch_ref(&branch.base_branch); - let siblings: Vec<String> = branches_in_clone(store, workspace, clone_dir.as_deref())? + let others: Vec<store::Branch> = branches_in_clone(store, workspace, clone_dir.as_deref())? .into_iter() .filter(|other| other.id != branch.id) - .map(|other| other.branch_name) .collect(); - let base_is_worked_on = siblings.contains(&base_ref); + let base_is_worked_on = others.iter().any(|other| other.branch_name == base_ref); + // Commits stranded on the base can only be the work of a branch that + // has had a session: one that never ran an agent committed nowhere. A + // branch added beside the one whose agent committed on the base would + // otherwise keep that one from taking its commits, and be kept from + // starting by it in turn. + let mut siblings = Vec::new(); + for other in others { + if store + .has_started_session_for_branch(&other.id) + .map_err(|e| e.to_string())? + { + siblings.push(other.branch_name); + } + } Ok(Checkout { workspace: workspace.to_string(), clone_dir, @@ -127,6 +140,7 @@ pub(super) async fn check_out_in_clone( /// A session found working in a clone. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LiveSession { + pub(crate) session_id: String, pub(crate) branch_name: String, /// `disconnected`: its turn may still be going on the workstation with /// no Staged process attached. @@ -193,7 +207,8 @@ pub(crate) fn live_session_in_clone( .find(|(_, session)| { (session.status == store::SessionStatus::Disconnected) == disconnected }) - .map(|(branch_name, _)| LiveSession { + .map(|(branch_name, session)| LiveSession { + session_id: session.id.clone(), branch_name: branch_name.clone(), disconnected, }) @@ -225,6 +240,7 @@ pub(crate) fn sibling_session_in_clone( .map_err(|e| e.to_string())?; if let Some(session) = busy.first() { return Ok(Some(LiveSession { + session_id: session.id.clone(), branch_name: sibling.branch_name, disconnected: session.status == store::SessionStatus::Disconnected, })); diff --git a/apps/staged/src-tauri/src/workspace_setup/failures.rs b/apps/staged/src-tauri/src/workspace_setup/failures.rs index f760fa3ea..b3c8f5076 100644 --- a/apps/staged/src-tauri/src/workspace_setup/failures.rs +++ b/apps/staged/src-tauri/src/workspace_setup/failures.rs @@ -17,9 +17,12 @@ //! fail into the card only, and a reason that has not changed is not //! announced again. //! -//! The same event carries the one thing a setup that worked has to say: a -//! `notice` that commits found on the base branch were brought along onto -//! the new branch ([`notify`]). It is not a reason and is not kept. +//! The same event carries what the user is to be told about a branch that +//! has no reason to show, as a `notice` ([`notify`]): that a setup which +//! worked brought commits found on the base branch along onto the new +//! branch, or, under a `noticeTitle` of its own, why the branch's queued +//! session is waiting ([`notify_queue_held`]). It is not a reason and is not +//! kept. //! //! In memory, per process: the branch row keeps only its status. After a //! restart the reasons are gone until the next attempt fails, which the @@ -43,6 +46,10 @@ struct SetupErrorEvent<'a> { user_requested: bool, #[serde(skip_serializing_if = "Option::is_none")] notice: Option<&'a str>, + /// The heading for `notice`, when it is not the one about commits + /// brought along. + #[serde(skip_serializing_if = "Option::is_none")] + notice_title: Option<&'a str>, } fn reasons() -> &'static Mutex<HashMap<String, String>> { @@ -70,6 +77,7 @@ pub(crate) fn set<R: tauri::Runtime>( error, user_requested: user_requested && error.is_some(), notice: None, + notice_title: None, }; crate::web_server::emit_to_all(app_handle, EVENT, event); }; @@ -103,6 +111,28 @@ pub(crate) fn notify<R: tauri::Runtime>( error: None, user_requested: false, notice: Some(notice), + notice_title: None, + }; + crate::web_server::emit_to_all(app_handle, EVENT, event); +} + +/// The heading of the notice [`notify_queue_held`] sends. +pub(crate) const QUEUE_HELD_TITLE: &str = "Queued session is waiting"; + +/// Tell every window and web client why a queued session of `branch_id` is +/// not starting. The branch is `Running` and has no reason to show, so +/// `error` is `null`, as for [`notify`]. +pub(crate) fn notify_queue_held<R: tauri::Runtime>( + app_handle: &tauri::AppHandle<R>, + branch_id: &str, + notice: &str, +) { + let event = SetupErrorEvent { + branch_id, + error: None, + user_requested: false, + notice: Some(notice), + notice_title: Some(QUEUE_HELD_TITLE), }; crate::web_server::emit_to_all(app_handle, EVENT, event); } diff --git a/apps/staged/src-tauri/src/workspace_setup/host.rs b/apps/staged/src-tauri/src/workspace_setup/host.rs index 56610da22..4a261e302 100644 --- a/apps/staged/src-tauri/src/workspace_setup/host.rs +++ b/apps/staged/src-tauri/src/workspace_setup/host.rs @@ -60,8 +60,8 @@ pub(crate) struct Checkout { /// Whether another branch in this clone is the base branch itself (see /// [`BranchCheckout::base_is_worked_on`]). pub(crate) base_is_worked_on: bool, - /// The other branches whose checkout is this clone (see - /// [`BranchCheckout::siblings`]). + /// The other branches whose checkout is this clone and that have had a + /// session (see [`BranchCheckout::siblings`]). pub(crate) siblings: Vec<String>, } diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs index 4dd519049..b6fc5bf52 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/checkout.rs @@ -6,14 +6,15 @@ use std::path::Path; use std::sync::Arc; use super::super::checkout::{local_branch_exists, BranchCheckout, CheckoutError}; +use super::super::clone::checkout_for; use super::super::SetupTrigger; -use super::verify::running_branch; +use super::verify::{running_branch, start_session_on}; use super::{ head_branch, origin_and_workstation_clone, set_up, starting_branch, status_of, store_with_project, FakeHost, Info, Start, }; use crate::branches; -use crate::store::WorkspaceStatus; +use crate::store::{self, Store, WorkspaceStatus}; use crate::test_utils::{run_git_in, TempDir}; fn rev(clone: &Path, name: &str) -> String { @@ -28,6 +29,16 @@ pub(super) fn commit(clone: &Path, file: &str, subject: &str) -> String { rev(clone, "HEAD") } +/// A session of `branch` that ran and is over: its agent may have committed +/// in the clone. +fn finished_session(store: &Store, branch: &store::Branch) { + let session = store::Session::new_running("work", Path::new("/tmp")).with_branch(&branch.id); + store.create_session(&session).unwrap(); + store + .update_session_status(&session.id, store::SessionStatus::Completed, None, None) + .unwrap(); +} + fn uncommitted(clone: &Path) -> Vec<String> { run_git_in(clone, &["status", "--porcelain"]) .lines() @@ -438,12 +449,13 @@ async fn setup_brings_the_commits_an_agent_left_on_the_base_branch() { /// on their branch in the workstation, `feature-b` starts from origin's /// base without them. #[tokio::test] -async fn two_branches_that_could_own_stranded_commits_are_not_guessed_between() { +async fn a_branch_is_not_given_commits_a_sibling_with_a_session_could_own() { let (_origin, clone) = origin_and_workstation_clone(); let origin_main = rev(clone.path(), "origin/main"); let stranded = commit(clone.path(), "one.txt", "agent work"); let (store, project) = store_with_project(); - let _first = running_branch(&store, &project, "feature-a"); + let first = running_branch(&store, &project, "feature-a"); + finished_session(&store, &first); let second = starting_branch(&store, &project, "feature-b"); let host = Arc::new(FakeHost::new( clone.path(), @@ -477,6 +489,97 @@ async fn two_branches_that_could_own_stranded_commits_are_not_guessed_between() assert_eq!(host.notices(), vec![]); } +/// Which siblings a checkout asks about: the ones that have had a session. +/// A branch with no session row, or only a queued one, never ran an agent +/// and made no commits. It still counts as work on the base branch when it +/// is the base branch. +#[test] +fn only_a_sibling_that_has_had_a_session_can_own_stranded_commits() { + let (store, project) = store_with_project(); + let branch = starting_branch(&store, &project, "feature-a"); + let added = starting_branch(&store, &project, "feature-b"); + let base = starting_branch(&store, &project, "main"); + + let checkout = checkout_for(&store, &branch).unwrap(); + assert_eq!(checkout.siblings, Vec::<String>::new()); + assert!( + checkout.base_is_worked_on, + "'main' is a branch in the clone" + ); + + let queued = store::Session::new_queued("note").with_branch(&added.id); + store.create_session(&queued).unwrap(); + assert_eq!( + checkout_for(&store, &branch).unwrap().siblings, + Vec::<String>::new(), + "a row that never left the queue ran nothing" + ); + + finished_session(&store, &added); + finished_session(&store, &base); + let mut siblings = checkout_for(&store, &branch).unwrap().siblings; + siblings.sort(); + assert_eq!(siblings, ["feature-b", "main"]); + assert_eq!( + checkout_for(&store, &added).unwrap().siblings, + ["main"], + "'feature-a' has had no session" + ); +} + +/// The incident with a second branch added beside it, which has never had a +/// session. Both used to end in `Error`: `feature-b` could not be set up +/// because `feature-a` has no local branch, and `feature-a` could not take +/// its commits because `feature-b` has none either. A branch that never ran +/// an agent can't have committed on the base, so it does not keep +/// `feature-a` from its commits. `feature-b` still waits for that: created +/// first, it would leave them on `main` with nothing to bring them along. +#[tokio::test] +async fn a_sibling_that_never_had_a_session_does_not_contest_stranded_commits() { + let (_origin, clone) = origin_and_workstation_clone(); + let origin_main = rev(clone.path(), "origin/main"); + let stranded = commit(clone.path(), "one.txt", "agent work"); + let (store, project) = store_with_project(); + let first = running_branch(&store, &project, "feature-a"); + finished_session(&store, &first); + let second = starting_branch(&store, &project, "feature-b"); + let queued = store::Session::new_queued("note").with_branch(&second.id); + store.create_session(&queued).unwrap(); + let host = Arc::new(FakeHost::new( + clone.path(), + &[Info::Status("running")], + Start::Succeeds, + )); + + let error = set_up(&store, &host, &second, SetupTrigger::Requested) + .await + .unwrap_err(); + assert!(error.contains("no branch 'feature-a' either"), "{error}"); + assert_eq!(head_branch(clone.path()), "main"); + assert_eq!(status_of(&store, &second), Some(WorkspaceStatus::Error)); + + // The next session of `feature-a` repairs its clone, and nothing of + // `feature-b`'s is in the way. + start_session_on(&store, &host, &first, "session-a") + .await + .unwrap(); + assert_eq!(head_branch(clone.path()), "feature-a"); + assert_eq!(rev(clone.path(), "HEAD"), stranded); + assert_eq!(status_of(&store, &first), Some(WorkspaceStatus::Running)); + let notices = host.notices(); + assert_eq!(notices.len(), 1, "{notices:?}"); + assert_eq!(notices[0].0, first.id); + + // Retry on `feature-b`: it starts from origin's base, without them. + set_up(&store, &host, &second, SetupTrigger::Requested) + .await + .unwrap(); + assert_eq!(head_branch(clone.path()), "feature-b"); + assert_eq!(rev(clone.path(), "HEAD"), origin_main); + assert_eq!(rev(clone.path(), "feature-a"), stranded); + assert_eq!(status_of(&store, &second), Some(WorkspaceStatus::Running)); +} + /// A branch that was never `Running` and can't be checked out without /// leaving commits behind fails where the user can see it, with the reason /// that names the branches and the commits. diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index 6d45487f5..250f872cf 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -340,7 +340,9 @@ fn only_setup_writes_running() { /// their branch, after that inspection, and leaves the rest `Starting`. /// Setup refuses a transferring branch, so the command asks for it once /// the transfer has ended without moving the project, before anything held -/// for the transfer is released. +/// for the transfer is released. A transfer that stops between the resume +/// and that inspection (a cancel, an inspection that fails) read nothing, +/// and leaves every row of a workspace that came back `Starting`. #[test] fn a_transfer_marks_running_only_the_rows_it_read_on_their_branch() { assert!( @@ -371,9 +373,29 @@ fn a_transfer_marks_running_only_the_rows_it_read_on_their_branch() { left_starting, "trigger_branch_setup(app_handle, store, branch_id, false);", ); + // The one other status write: `Starting`, on the two ways out between + // the resume and the inspection that follows it. + let starting = body_of(TRANSFER, " fn mark_resumed_rows_starting("); + assert_calls( + "mark_resumed_rows_starting", + starting, + "update_branch_workspace_status(&source.branch.id, &WorkspaceStatus::Starting)", + ); + assert!(!starting.contains("WorkspaceStatus::Running")); + let inspect = body_of( + TRANSFER, + " fn inspect(&self) -> Result<Inspection, Failure> {", + ); + assert_eq!( + inspect + .matches("self.mark_resumed_rows_starting(&inspection, &resumed);") + .count(), + 2, + "a cancel after a resume and a failed second inspection both leave the rows Starting" + ); assert_eq!( TRANSFER.matches("update_branch_workspace_status(").count(), - 1, + 2, "a new status write in the transfer needs its own reading of the clone" ); assert_eq!( @@ -451,11 +473,11 @@ fn every_session_start_checks_the_clone_before_it_launches() { /// The pipelines that act on HEAD (commit, rebase, squash, pull) run git /// in the clone without an agent, and get the same check before their first -/// step: before the row is claimed or inserted as running. The pull request -/// pipeline makes it too, for the steps that describe the branch, and goes -/// on whatever it says. A push does not make it: it names its local branch -/// in an explicit refspec, and used to fail while a sibling branch's -/// session had the clone. +/// step: before the row is claimed or inserted as running. A push does not +/// make it: it names its local branch in an explicit refspec, and used to +/// fail while a sibling branch's session had the clone. Neither does the +/// pull request pipeline, whose steps and prompts name the local branch and +/// whose first step stops it when the clone has no such branch. #[test] fn every_pipeline_checks_the_clone_before_its_git_steps() { let starts = [ @@ -474,11 +496,6 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { "pub(crate) async fn start_queued_git_pipeline_for_branch(", "claim.claim(&store)", ), - ( - PR_CREATE, - "pub(crate) async fn start_create_pr_pipeline_for_branch(", - "start_pipeline_for_branch(", - ), ]; for (source, start, launch) in starts { let body = body_of(source, start); @@ -495,7 +512,7 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { } // A push is not held up by where the clone is, run now or from the - // queue, and the pull request pipeline never fails on it. + // queue, and the pull request pipeline does not look. let push = body_of( PR_PUSH, "pub(crate) async fn start_or_queue_push_pipeline_for_branch(", @@ -519,8 +536,19 @@ fn every_pipeline_checks_the_clone_before_its_git_steps() { "pub(crate) async fn start_create_pr_pipeline_for_branch(", ); assert!( - !create_pr.contains(".await?") && create_pr.contains(".unwrap_or_else(|e| {"), - "the pull request pipeline must not fail on the checkout check" + !PR_CREATE.contains("ensure_pipeline_checkout(") + && !PR_CREATE.contains("ensure_session_checkout(") + && !create_pr.contains("CheckoutHold"), + "the pull request pipeline must not check or repair the clone's HEAD" + ); + // What stands in for the check: the step that stops the pipeline when + // the clone has no local branch comes before every other step. + let steps = body_of(PR_CREATE, "pub(super) fn build_create_pr_pipeline_steps("); + assert_before( + "build_create_pr_pipeline_steps", + steps, + "command: check_local_branch_command(local_branch),\n on_failure: FailureStrategy::Abort { marker: None },", + "label: \"Fetch latest base\".to_string(),", ); // The pull: checked in the task that runs it, before the pull itself. @@ -571,19 +599,30 @@ fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { assert_before( "drain_branch_queue", still_queued, - "hold_queue_behind_sibling_session(&store, kicker, &branch, &session_id)", + "hold_queue_behind_sibling_session(", "retry_or_fail_queued_start_before_claim(", ); assert_before( "drain_branch_queue", drain, - "kicker.start_attempts().is_held(&branch_id, &session.id)\n && hold_queue_behind_sibling_session(&store, kicker, &branch, &session.id)", + "kicker.start_attempts().is_held(&branch_id, &session.id)\n && hold_queue_behind_sibling_session(", "start_queued_session_for_branch(", ); - let hold = body_of(SESSION_COMMANDS, "fn hold_queue_behind_sibling_session("); + // The hold tells the user why the row waits and has a `disconnected` + // holder looked at, on the first look and on each recheck alike: both + // drain call sites go through the one function. + let hold = body_of(SESSION_COMMANDS, "fn hold_queue_behind_sibling_session<"); + for call in [ + "sibling_session_hold(store, kicker, branch, session_id)", + "crate::workspace_setup::failures::notify_queue_held(app_handle, &branch.id, notice)", + "crate::session_runner::reconcile::reconcile_session(holder.clone())", + ] { + assert_calls("hold_queue_behind_sibling_session", hold, call); + } + let look = body_of(SESSION_COMMANDS, "fn sibling_session_hold("); assert_calls( - "hold_queue_behind_sibling_session", - hold, + "sibling_session_hold", + look, "crate::workspace_setup::clone::sibling_session_in_clone(store, branch)", ); } diff --git a/apps/staged/src-tauri/src/workspace_setup/verify.rs b/apps/staged/src-tauri/src/workspace_setup/verify.rs index d8d72de22..4e9b422f8 100644 --- a/apps/staged/src-tauri/src/workspace_setup/verify.rs +++ b/apps/staged/src-tauri/src/workspace_setup/verify.rs @@ -9,8 +9,10 @@ //! ([`confirm_checked_out`]); //! - at every start on a remote branch of something that runs git in the //! clone, before it is launched ([`ensure_session_checkout`]): an agent -//! session, fresh or resumed, and a pipeline session (commit, push, -//! rebase, squash, PR). A branch can be `Running` over a clone that has +//! session, fresh or resumed, and a pipeline session that acts on HEAD +//! (commit, rebase, squash, pull). A push and the pull request pipeline +//! name their local branch and make no check. A branch can be `Running` +//! over a clone that has //! since moved: two branches of one repo share a clone, someone switches //! branches in a terminal. //! diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index 8bc97595b..7793a332b 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -17,12 +17,15 @@ interface LocalSetupTask { * `userRequested` marks the failure of a setup the user asked for (a new * branch, Retry, Resume), as opposed to one the backend ran on its own. * `notice` comes with a setup that worked and did something the user - * should know of: it put commits it found on the base branch on this one. */ + * should know of: it put commits it found on the base branch on this one. + * With a `noticeTitle` it is about something else, under that heading: why + * the branch's queued session is waiting. */ interface WorkspaceSetupErrorEvent { branchId: string; error: string | null; userRequested?: boolean; notice?: string; + noticeTitle?: string; } interface WorkspaceLifecycleHooks { @@ -354,7 +357,7 @@ class WorkspaceLifecycleController { if (error == null) { this.clearWorkspaceError(branchId); if (event.notice) { - toast.warning('Workspace branch was given existing commits', { + toast.warning(event.noticeTitle ?? 'Workspace branch was given existing commits', { description: event.notice, duration: Infinity, }); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 0d5db7f3d..ffdd029f0 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -487,6 +487,31 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); + // A queued session held behind a sibling branch's session stays queued + // with no error. The backend says why on the same event, under a heading + // of its own, and the branch is given no reason to show. + it('tells the user why a queued session is waiting, under the heading it came with', async () => { + const lifecycle = await newController(); + await settle(); + + const notice = + "A queued session of branch 'feature' is waiting: a session of branch 'sibling' is " + + 'running in the same clone. It starts once that session has finished.'; + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + userRequested: false, + notice, + noticeTitle: 'Queued session is waiting', + }); + expect(toastWarning.mock.calls).toEqual([ + ['Queued session is waiting', { description: notice, duration: Infinity }], + ]); + expect(toastError).not.toHaveBeenCalled(); + expect(lifecycle.getWorkspaceErrors().has('branch-1')).toBe(false); + lifecycle.stop(); + }); + // The backend reports every failed run, and a snapshot covers the ones // from before the window mounted. it('keeps setup reasons from the error event and the snapshot taken at start', async () => { diff --git a/apps/staged/src/lib/listeners/sessionStatusListener.test.ts b/apps/staged/src/lib/listeners/sessionStatusListener.test.ts index 50d94f50c..f71e9816d 100644 --- a/apps/staged/src/lib/listeners/sessionStatusListener.test.ts +++ b/apps/staged/src/lib/listeners/sessionStatusListener.test.ts @@ -1529,6 +1529,25 @@ describe('sessionStatusListener busy-state hydration', () => { ); }); + // The pull request pipeline stops itself, before any agent, when the + // checkout has no local branch to push. The backend ends the session in + // `error` with the reason, which is what tells the user what to do. + it('shows why a pr session failed when the backend says so', async () => { + await listen(); + registerSession('pr-1', 'pr'); + const reason = + "Check local branch failed:\n\nLocal branch 'feature' does not exist in this clone."; + + eventCallbacks.get('session-status-changed')?.({ + sessionId: 'pr-1', + status: 'error', + errorMessage: reason, + }); + + expect(prStateStore.setPrError).toHaveBeenCalledWith('branch-1', reason); + expect(prStateStore.setPrError).toHaveBeenCalledTimes(1); + }); + it('falls back to success rendering when the push-completed event was missed', async () => { await listen(); registerSession('push-1', 'push'); diff --git a/apps/staged/src/lib/listeners/sessionStatusListener.ts b/apps/staged/src/lib/listeners/sessionStatusListener.ts index 59b3cd2ec..8357ae0b3 100644 --- a/apps/staged/src/lib/listeners/sessionStatusListener.ts +++ b/apps/staged/src/lib/listeners/sessionStatusListener.ts @@ -670,7 +670,7 @@ function handleSessionEnd( } if (sessionType === 'pr' && branchId) { - handlePrCompletion(branchId, status); + handlePrCompletion(branchId, status, errorMessage); prStateStore.clearSessionTracking(branchId); } @@ -734,8 +734,13 @@ function handlePullCompletion( * successful completion the branch is already marked created by the time * this runs; a completed session whose branch is still not created means no * PR URL was found in the output. + * + * A session that ended in `error` with a message says why in the PR error + * dialog: the pipeline stops itself, before any agent, when the checkout has + * no local branch to push (see `prs/create_pr.rs`), and that reason is the + * only thing that tells the user what to do. */ -function handlePrCompletion(branchId: string, status: SessionStatus) { +function handlePrCompletion(branchId: string, status: SessionStatus, errorMessage?: string | null) { if (status === 'completed') { if (prStateStore.getPrState(branchId)?.state !== 'created') { prStateStore.setPrError( @@ -743,6 +748,8 @@ function handlePrCompletion(branchId: string, status: SessionStatus) { 'PR session completed but no PR URL was found in the output.' ); } + } else if (status === 'error' && errorMessage) { + prStateStore.setPrError(branchId, errorMessage); } else { prStateStore.setPrError( branchId, From d74fa744a040f8827e4ba48a6a226e7bdaa97164 Mon Sep 17 00:00:00 2001 From: Matt Toohey <contact@matttoohey.com> Date: Thu, 19 Nov 2026 12:04:22 +1100 Subject: [PATCH 7/9] fix(staged): ask about a disconnected holder once per cause and take the hold toast down on release Applies the one warning and the one suggestion of the review of 89e135d5, and nothing else. Reconcile throttling. A queued row held behind a sibling branch's `disconnected` session asked the reconcile service on every look, with the "Check now" trigger that skips every spacing the service keeps, and the hold is looked at again every five seconds. That was a Blox call, a log line and two events every five seconds for as long as the hold lasted. The holder is now named for the service only when the cause is new: on the first look, and when the holder goes from running to disconnected. The rechecks are a database read again, and the service's own sweep, held-work recheck and workspace-reachable nudge look at the session after that. The comment on `QUEUE_HOLD_RECHECK` says so. Hold toast. The "Queued session is waiting" toast had no id and nothing took it down. It now has one id per branch, so a new cause replaces the old toast instead of stacking another. The backend takes the notice back when the hold ends: when a look finds nothing holding the row, and when a held row has left the queue (stopped or force-started). It sends the same event with the heading and no notice, and the frontend dismisses the toast on it. Clearing a branch's state dismisses it too. The adopted-commits notice is unchanged and stays sticky. `QueuedStartAttempts` gains `release`, and `retain_queued` reports whether it dropped a held row. No data model changes and no new event. Tests: the hold test now pins one ask per cause across rechecks and a running/disconnected flap, and one take-back event on release. The attempts test covers `release` and the pruned held row, the wiring test pins both take-back call sites, and the vitest file covers the toast id, the replacement and both dismissals. Signed-off-by: Matt Toohey <contact@matttoohey.com> Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com> --- apps/staged/src-tauri/src/session_commands.rs | 77 ++++++++++++++----- .../src-tauri/src/session_queue/retry.rs | 32 +++++++- .../src-tauri/src/session_queue/tests.rs | 14 +++- .../src-tauri/src/workspace_setup/failures.rs | 23 +++++- .../src/workspace_setup/tests/wiring.rs | 12 ++- .../projects/workspaceLifecycle.svelte.ts | 22 +++++- .../projects/workspaceLifecycle.test.ts | 39 ++++++++-- 7 files changed, 183 insertions(+), 36 deletions(-) diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index 29628bf42..c8a5f5c76 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -4528,10 +4528,12 @@ fn retry_or_fail_queued_start_before_claim<R: tauri::Runtime>( /// /// A held row stays `queued` with no error, so the user is told why it /// waits, once per cause, as a notice on the branch's setup event -/// (`workspace_setup::failures::notify_queue_held`). And a `disconnected` -/// holder is handed to the reconcile service on every look, this one and -/// each recheck: left to the service's own backoff it could hold the queue -/// for half an hour after its turn was over. +/// (`workspace_setup::failures::notify_queue_held`), and that notice is +/// taken back when a look finds the row no longer held. A `disconnected` +/// holder is handed to the reconcile service once per cause as well, not on +/// each recheck: asking skips every spacing that service keeps, and a +/// recheck comes every few seconds for as long as the hold lasts. After the +/// first ask the service's own sweep and nudges look at the session. fn hold_queue_behind_sibling_session<R: tauri::Runtime>( store: &Store, app_handle: &tauri::AppHandle<R>, @@ -4540,6 +4542,9 @@ fn hold_queue_behind_sibling_session<R: tauri::Runtime>( session_id: &str, ) -> bool { let Some(hold) = sibling_session_hold(store, kicker, branch, session_id) else { + if kicker.start_attempts().release(&branch.id, session_id) { + crate::workspace_setup::failures::notify_queue_released(app_handle, &branch.id); + } return false; }; if let Some(notice) = &hold.announce { @@ -4558,8 +4563,8 @@ fn hold_queue_behind_sibling_session<R: tauri::Runtime>( struct SiblingHold { /// Why the row waits, when the user has not been told this yet. announce: Option<String>, - /// The session holding the clone, when it is `disconnected`: the - /// reconcile service is to look at it now. + /// The session holding the clone, when it is `disconnected` and that + /// is news: the reconcile service is to look at it now. disconnected_holder: Option<String>, } @@ -4608,7 +4613,7 @@ fn sibling_session_hold( ); Some(SiblingHold { announce: news.then_some(reason), - disconnected_holder: live.disconnected.then_some(live.session_id), + disconnected_holder: (news && live.disconnected).then_some(live.session_id), }) } @@ -6459,9 +6464,14 @@ pub(crate) async fn drain_branch_queue( .get_queued_sessions_for_branch(&branch_id) .map_err(|e| e.to_string())?; if let Some(kicker) = &kicker { - kicker + // A held row that left the queue (stopped, forced) is not waiting + // any more: its notice goes with it. + if kicker .start_attempts() - .retain_queued(&branch_id, |id| queued.iter().any(|s| s.id == id)); + .retain_queued(&branch_id, |id| queued.iter().any(|s| s.id == id)) + { + crate::workspace_setup::failures::notify_queue_released(&app_handle, &branch_id); + } } let mut active = running_branch_session_kinds(&store, &branch_id)?; @@ -12727,10 +12737,12 @@ mod tests { /// A held row stays `queued` with no error, and its reason used to go to /// the log alone. The user is told why it waits as a notice on the /// branch's setup event, once per cause and not on every recheck. A - /// holder that is `disconnected` is named for the reconcile service on - /// every look, so the hold does not wait out that service's backoff. + /// holder that is `disconnected` is named for the reconcile service once + /// per cause too: a recheck comes every few seconds, and asking on each + /// would skip the spacing that service keeps between its Blox calls. + /// When nothing holds the row any more the notice is taken back, once. #[test] - fn a_held_queue_says_why_once_and_has_a_disconnected_holder_checked_on_every_look() { + fn a_held_queue_says_why_and_has_a_disconnected_holder_checked_once_per_cause() { use crate::session_queue::BranchQueueKicker; use tauri::Listener; @@ -12781,7 +12793,7 @@ mod tests { assert_eq!(row.error_message, None); // The holder loses its connection: a new cause, told once, and the - // session is named for the reconcile service on each look. + // session is named for the reconcile service on that look alone. store .update_session_status(&agent.id, store::SessionStatus::Disconnected, None, None) .unwrap(); @@ -12799,19 +12811,48 @@ mod tests { look(), SiblingHold { announce: None, - disconnected_holder: Some(agent.id.clone()), + disconnected_holder: None, } ); } - // Settled: nothing holds the row, and nobody is asked about. + // Back to running and lost again: each is a new cause, and the + // second loss is asked about once more. store - .update_session_status(&agent.id, store::SessionStatus::Completed, None, None) + .update_session_status(&agent.id, store::SessionStatus::Running, None, None) + .unwrap(); + assert_eq!(look().disconnected_holder, None); + store + .update_session_status(&agent.id, store::SessionStatus::Disconnected, None, None) .unwrap(); assert_eq!( - sibling_session_hold(&store, &kicker, &branch, &first.id), - None + look().disconnected_holder.as_deref(), + Some(agent.id.as_str()) ); + assert_eq!(look().disconnected_holder, None); + + // Settled: nothing holds the row, nobody is asked about, and the + // notice is taken back with the same heading and no text, once. + store + .update_session_status(&agent.id, store::SessionStatus::Completed, None, None) + .unwrap(); + let told_while_held = events.lock().unwrap().len(); + for _ in 0..2 { + assert!(!hold_queue_behind_sibling_session( + &store, + app.handle(), + &kicker, + &branch, + &first.id + )); + } + let told = events.lock().unwrap().clone(); + assert_eq!(told.len(), told_while_held + 1, "{told:?}"); + let event: serde_json::Value = serde_json::from_str(told.last().unwrap()).unwrap(); + assert_eq!(event["branchId"], branch.id.as_str()); + assert_eq!(event["noticeTitle"], "Queued session is waiting"); + assert_eq!(event.get("notice"), None); + assert!(!kicker.start_attempts().is_held(&branch.id, &first.id)); } /// A pre-claim start failure is retried before it is recorded: the first diff --git a/apps/staged/src-tauri/src/session_queue/retry.rs b/apps/staged/src-tauri/src/session_queue/retry.rs index 8df670998..af502e69e 100644 --- a/apps/staged/src-tauri/src/session_queue/retry.rs +++ b/apps/staged/src-tauri/src/session_queue/retry.rs @@ -59,8 +59,10 @@ pub(crate) fn queued_start_retry(failures: u32) -> QueuedStartRetry { } /// How long a held row waits before the drain looks at what holds it again. -/// The look is a database read, so it is short: the queue moves within -/// seconds of the cause going away. +/// A recheck is a database read and nothing more, so it is short: the queue +/// moves within seconds of the cause going away. What costs a Blox call, +/// asking the reconcile service about a `disconnected` holder, is done once +/// per cause and not on each recheck. pub(crate) const QUEUE_HOLD_RECHECK: Duration = Duration::from_secs(5); struct Attempt { @@ -140,6 +142,17 @@ impl QueuedStartAttempts { news } + /// End `session_id`'s hold, keeping its attempts. Returns whether it + /// was held, which is when the user was told it waits. + pub(crate) fn release(&self, branch_id: &str, session_id: &str) -> bool { + self.by_branch + .lock() + .unwrap() + .get_mut(branch_id) + .and_then(|rows| rows.get_mut(session_id)) + .is_some_and(|attempt| attempt.held.take().is_some()) + } + /// Whether `session_id` was last left where it is by [`Self::hold`]. pub(crate) fn is_held(&self, branch_id: &str, session_id: &str) -> bool { self.by_branch @@ -161,14 +174,25 @@ impl QueuedStartAttempts { } /// Drop the branch's entries for rows that are no longer queued. - pub(crate) fn retain_queued(&self, branch_id: &str, still_queued: impl Fn(&str) -> bool) { + /// Returns whether a held row was among them. + pub(crate) fn retain_queued( + &self, + branch_id: &str, + still_queued: impl Fn(&str) -> bool, + ) -> bool { let mut by_branch = self.by_branch.lock().unwrap(); + let mut dropped_held = false; if let Some(rows) = by_branch.get_mut(branch_id) { - rows.retain(|session_id, _| still_queued(session_id)); + rows.retain(|session_id, attempt| { + let keep = still_queued(session_id); + dropped_held |= !keep && attempt.held.is_some(); + keep + }); if rows.is_empty() { by_branch.remove(branch_id); } } + dropped_held } /// Drop every entry for a branch that is gone. diff --git a/apps/staged/src-tauri/src/session_queue/tests.rs b/apps/staged/src-tauri/src/session_queue/tests.rs index dc32e7d09..d6cfd30fc 100644 --- a/apps/staged/src-tauri/src/session_queue/tests.rs +++ b/apps/staged/src-tauri/src/session_queue/tests.rs @@ -326,9 +326,19 @@ fn queued_start_attempts_retry_with_growing_delays_then_give_up() { // A row that left the queue (stopped, forced, deleted) is forgotten on // the next drain's prune; the branch's map goes with its last row. - attempts.retain_queued("b", |id| id == "s2"); + assert!(!attempts.retain_queued("b", |id| id == "s2")); assert_eq!(attempts.failures("b", "s2"), 1); - attempts.retain_queued("b", |_| false); + + // A hold ends once, keeping the row's attempts; a held row that left + // the queue is reported, so what the user was told can be taken back. + assert!(!attempts.release("b", "s2")); + attempts.hold("b", "s2", t2, "waiting"); + assert!(attempts.release("b", "s2")); + assert!(!attempts.release("b", "s2")); + assert!(!attempts.is_held("b", "s2")); + assert_eq!(attempts.failures("b", "s2"), 1); + attempts.hold("b", "s2", t2, "waiting"); + assert!(attempts.retain_queued("b", |_| false)); assert!(attempts.is_empty()); attempts.record_failure("gone", "s3", t2); diff --git a/apps/staged/src-tauri/src/workspace_setup/failures.rs b/apps/staged/src-tauri/src/workspace_setup/failures.rs index b3c8f5076..9c307319b 100644 --- a/apps/staged/src-tauri/src/workspace_setup/failures.rs +++ b/apps/staged/src-tauri/src/workspace_setup/failures.rs @@ -22,7 +22,8 @@ //! worked brought commits found on the base branch along onto the new //! branch, or, under a `noticeTitle` of its own, why the branch's queued //! session is waiting ([`notify_queue_held`]). It is not a reason and is not -//! kept. +//! kept. A `noticeTitle` with no `notice` takes that notice back +//! ([`notify_queue_released`]). //! //! In memory, per process: the branch row keeps only its status. After a //! restart the reasons are gone until the next attempt fails, which the @@ -126,12 +127,30 @@ pub(crate) fn notify_queue_held<R: tauri::Runtime>( app_handle: &tauri::AppHandle<R>, branch_id: &str, notice: &str, +) { + emit_queue_hold(app_handle, branch_id, Some(notice)); +} + +/// Tell every window and web client that the queued session of `branch_id` +/// no longer waits, so what [`notify_queue_held`] said is taken down: the +/// same heading with no `notice`. +pub(crate) fn notify_queue_released<R: tauri::Runtime>( + app_handle: &tauri::AppHandle<R>, + branch_id: &str, +) { + emit_queue_hold(app_handle, branch_id, None); +} + +fn emit_queue_hold<R: tauri::Runtime>( + app_handle: &tauri::AppHandle<R>, + branch_id: &str, + notice: Option<&str>, ) { let event = SetupErrorEvent { branch_id, error: None, user_requested: false, - notice: Some(notice), + notice, notice_title: Some(QUEUE_HELD_TITLE), }; crate::web_server::emit_to_all(app_handle, EVENT, event); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index 250f872cf..39594160f 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -609,12 +609,20 @@ fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { "start_queued_session_for_branch(", ); // The hold tells the user why the row waits and has a `disconnected` - // holder looked at, on the first look and on each recheck alike: both - // drain call sites go through the one function. + // holder looked at, once per cause, and takes the notice back when the + // row is no longer held: both drain call sites go through the one + // function. A held row that left the queue has it taken back too. + assert_calls( + "drain_branch_queue", + drain, + "crate::workspace_setup::failures::notify_queue_released(&app_handle, &branch_id)", + ); let hold = body_of(SESSION_COMMANDS, "fn hold_queue_behind_sibling_session<"); for call in [ "sibling_session_hold(store, kicker, branch, session_id)", "crate::workspace_setup::failures::notify_queue_held(app_handle, &branch.id, notice)", + "kicker.start_attempts().release(&branch.id, session_id)", + "crate::workspace_setup::failures::notify_queue_released(app_handle, &branch.id)", "crate::session_runner::reconcile::reconcile_session(holder.clone())", ] { assert_calls("hold_queue_behind_sibling_session", hold, call); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index 7793a332b..70b6ffc71 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -19,7 +19,8 @@ interface LocalSetupTask { * `notice` comes with a setup that worked and did something the user * should know of: it put commits it found on the base branch on this one. * With a `noticeTitle` it is about something else, under that heading: why - * the branch's queued session is waiting. */ + * the branch's queued session is waiting. That heading with no `notice` + * says the session no longer waits. */ interface WorkspaceSetupErrorEvent { branchId: string; error: string | null; @@ -28,6 +29,12 @@ interface WorkspaceSetupErrorEvent { noticeTitle?: string; } +/** The one toast a branch has for why its queued session is waiting: a new + * cause replaces the old one, and it is taken down when the wait is over. */ +function queueHeldToastId(branchId: string): string { + return `queue-held:${branchId}`; +} + interface WorkspaceLifecycleHooks { getBranchesByProject: () => BranchMap; setBranchesByProject: (next: BranchMap) => void; @@ -302,6 +309,7 @@ class WorkspaceLifecycleController { this.pendingSetupBranches.delete(branchId); this.queuedSetupBranches.delete(branchId); this.prerunOnSetup.delete(branchId); + toast.dismiss(queueHeldToastId(branchId)); this.localSetupTaskQueue = this.localSetupTaskQueue.filter( (task) => task.branchId !== branchId @@ -356,8 +364,16 @@ class WorkspaceLifecycleController { this.setupErrorEventSeqByBranch.set(branchId, ++this.setupErrorEventSeq); if (error == null) { this.clearWorkspaceError(branchId); - if (event.notice) { - toast.warning(event.noticeTitle ?? 'Workspace branch was given existing commits', { + if (event.noticeTitle) { + // Why the branch's queued session waits: up for as long as it does. + const id = queueHeldToastId(branchId); + if (event.notice) { + toast.warning(event.noticeTitle, { id, description: event.notice, duration: Infinity }); + } else { + toast.dismiss(id); + } + } else if (event.notice) { + toast.warning('Workspace branch was given existing commits', { description: event.notice, duration: Infinity, }); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index ffdd029f0..7b14f5c5a 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -13,6 +13,7 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { let eventListeners: Map<string, Array<(payload: unknown) => void>>; let toastError: ReturnType<typeof vi.fn>; let toastWarning: ReturnType<typeof vi.fn>; + let toastDismiss: ReturnType<typeof vi.fn>; /** Deliver a backend event to the controller's listeners. */ function emit(event: string, payload: unknown) { @@ -62,7 +63,8 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { vi.doMock('svelte-sonner', () => { toastError = vi.fn(); toastWarning = vi.fn(); - return { toast: { error: toastError, warning: toastWarning } }; + toastDismiss = vi.fn(); + return { toast: { error: toastError, warning: toastWarning, dismiss: toastDismiss } }; }); vi.doMock('../settings/preferences.svelte', () => ({ getPreferredAgent: () => 'claude', @@ -481,16 +483,21 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { ]); expect(toastError).not.toHaveBeenCalled(); - // A reason cleared with nothing to tell raises nothing. + // A reason cleared with nothing to tell raises nothing, and this notice + // is not one that is taken down. emit('workspace-setup-error', { branchId: 'branch-1', error: null }); expect(toastWarning).toHaveBeenCalledTimes(1); + expect(toastDismiss).not.toHaveBeenCalled(); lifecycle.stop(); }); // A queued session held behind a sibling branch's session stays queued // with no error. The backend says why on the same event, under a heading - // of its own, and the branch is given no reason to show. - it('tells the user why a queued session is waiting, under the heading it came with', async () => { + // of its own, and the branch is given no reason to show. The toast has + // one id per branch, so a new cause replaces the old one, and it is taken + // down when the backend says the session no longer waits, or when the + // branch's state is cleared. + it('tells the user why a queued session is waiting, for as long as it waits', async () => { const lifecycle = await newController(); await settle(); @@ -504,11 +511,33 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { notice, noticeTitle: 'Queued session is waiting', }); + const lost = "A queued session of branch 'feature' is waiting: lost its connection."; + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + notice: lost, + noticeTitle: 'Queued session is waiting', + }); + const id = 'queue-held:branch-1'; expect(toastWarning.mock.calls).toEqual([ - ['Queued session is waiting', { description: notice, duration: Infinity }], + ['Queued session is waiting', { id, description: notice, duration: Infinity }], + ['Queued session is waiting', { id, description: lost, duration: Infinity }], ]); expect(toastError).not.toHaveBeenCalled(); + expect(toastDismiss).not.toHaveBeenCalled(); expect(lifecycle.getWorkspaceErrors().has('branch-1')).toBe(false); + + // Released: the heading alone takes the toast down and raises nothing. + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + noticeTitle: 'Queued session is waiting', + }); + expect(toastDismiss.mock.calls).toEqual([[id]]); + expect(toastWarning).toHaveBeenCalledTimes(2); + + lifecycle.clearBranchState('branch-2'); + expect(toastDismiss).toHaveBeenLastCalledWith('queue-held:branch-2'); lifecycle.stop(); }); From baaa2ee98749fd5a44e907b3bce7e00c4cfc6b0e Mon Sep 17 00:00:00 2001 From: Matt Toohey <contact@matttoohey.com> Date: Thu, 19 Nov 2026 12:36:54 +1100 Subject: [PATCH 8/9] fix(staged): bound a held queue's wait on a disconnected holder and release the hold toast with its branch Applies the warning and the suggestion of the review of f8e6a82e, and nothing else. The two other missed-release cases that review lists (a branch that is not Running or is being transferred, and a backend restart under an open web client) are left as they are. Bounded wait. A queued row held behind a sibling branch's `disconnected` session asked the reconcile service once per cause, and then waited for that service's own schedule. When the holder's attempts had been failing, that schedule is a backoff of up to thirty minutes, so the row could wait about 31 minutes after the cause had cleared. A second disconnected session of the same sibling branch becoming the holder was not asked about at all, because the reason text was the same. The hold now has its own reconcile trigger, `Trigger::QueueHold`, sent through `reconcile::note_queue_held_behind` on every look at the hold, for whichever session is the holder. The service lets it start an attempt only when the holder row's last attempt began at least `QUEUE_HOLD_MIN_SPACING` ago, whatever its backoff says. So a held row costs at most one attempt per spacing, and it waits at most that spacing plus one five-second recheck once the cause is over. Every other ask is a map lookup. The spacing is one minute. That is the service's own `BACKOFF_BASE` and `HELD_WORK_RECHECK`, the spacing it keeps by itself after a row's first failed attempt, so the hold never has a row tried more often than the service already does. It only stops that spacing from growing while a queue waits on the answer. One side effect: a holder that is still working after its turn is checked every minute while a queue waits on it, where the service alone would space those checks out to five. The comment on `QUEUE_HOLD_RECHECK` says what a recheck now does. Hold toast. When the drain finds a branch gone it forgets the branch's attempts and sent nothing, so only the window that deleted the branch dropped the "Queued session is waiting" toast. `forget_branch` now reports whether a held row was among what it dropped, as `retain_queued` does, and the drain sends the release event for it. The frontend compares the notice heading against the hold's heading instead of taking any heading for the hold notice. A notice under another heading is shown under that heading without the hold toast's id, and such a heading alone does nothing. No data model changes and no new event. Tests: a reconcile test pins one attempt per spacing whatever the backoff, and the settle once the cause clears. The hold test now pins that a disconnected holder is named on every look, and that a second disconnected session of the same sibling is named with no new notice. The attempts test covers what `forget_branch` reports, the wiring test pins both take-back call sites in the drain and the new trigger call, and the vitest file covers a notice under another heading. Signed-off-by: Matt Toohey <contact@matttoohey.com> Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com> --- apps/staged/src-tauri/src/session_commands.rs | 68 +++++++++++++------ .../src-tauri/src/session_queue/retry.rs | 20 ++++-- .../src-tauri/src/session_queue/tests.rs | 8 ++- .../src-tauri/src/session_runner/reconcile.rs | 33 ++++++++- .../src/session_runner/reconcile/tests.rs | 54 +++++++++++++++ .../src/workspace_setup/tests/wiring.rs | 30 +++++--- .../projects/workspaceLifecycle.svelte.ts | 10 ++- .../projects/workspaceLifecycle.test.ts | 27 ++++++++ 8 files changed, 208 insertions(+), 42 deletions(-) diff --git a/apps/staged/src-tauri/src/session_commands.rs b/apps/staged/src-tauri/src/session_commands.rs index c8a5f5c76..37b5e813c 100644 --- a/apps/staged/src-tauri/src/session_commands.rs +++ b/apps/staged/src-tauri/src/session_commands.rs @@ -4530,10 +4530,12 @@ fn retry_or_fail_queued_start_before_claim<R: tauri::Runtime>( /// waits, once per cause, as a notice on the branch's setup event /// (`workspace_setup::failures::notify_queue_held`), and that notice is /// taken back when a look finds the row no longer held. A `disconnected` -/// holder is handed to the reconcile service once per cause as well, not on -/// each recheck: asking skips every spacing that service keeps, and a -/// recheck comes every few seconds for as long as the hold lasts. After the -/// first ask the service's own sweep and nudges look at the session. +/// holder is named for the reconcile service on every look, with a trigger +/// of the hold's own (`reconcile::note_queue_held_behind`): the service +/// tries the session only when its last attempt is a minute old, so the +/// rechecks cost one Blox call a minute at most, and the row waits no +/// longer than that once whatever kept the holder from settling is over, +/// instead of as long as the service's failure backoff had grown. fn hold_queue_behind_sibling_session<R: tauri::Runtime>( store: &Store, app_handle: &tauri::AppHandle<R>, @@ -4550,10 +4552,8 @@ fn hold_queue_behind_sibling_session<R: tauri::Runtime>( if let Some(notice) = &hold.announce { crate::workspace_setup::failures::notify_queue_held(app_handle, &branch.id, notice); } - if let Some(holder) = hold.disconnected_holder { - if let Err(e) = crate::session_runner::reconcile::reconcile_session(holder.clone()) { - log::debug!("[session_queue] could not ask for a check of session {holder}: {e}"); - } + if let Some(holder) = &hold.disconnected_holder { + crate::session_runner::reconcile::note_queue_held_behind(holder); } true } @@ -4563,8 +4563,8 @@ fn hold_queue_behind_sibling_session<R: tauri::Runtime>( struct SiblingHold { /// Why the row waits, when the user has not been told this yet. announce: Option<String>, - /// The session holding the clone, when it is `disconnected` and that - /// is news: the reconcile service is to look at it now. + /// The session holding the clone, when it is `disconnected`: the + /// reconcile service is told the queue waits on it, on every look. disconnected_holder: Option<String>, } @@ -4613,7 +4613,7 @@ fn sibling_session_hold( ); Some(SiblingHold { announce: news.then_some(reason), - disconnected_holder: (news && live.disconnected).then_some(live.session_id), + disconnected_holder: live.disconnected.then_some(live.session_id), }) } @@ -6446,8 +6446,12 @@ pub(crate) async fn drain_branch_queue( .map(|kicker| Arc::clone(&kicker)); let Some(branch) = store.get_branch(&branch_id).map_err(|e| e.to_string())? else { log::debug!("[session_queue] branch {branch_id} is gone; nothing to drain"); + // A held row went with its branch. The window that deleted it has + // dropped the notice; every other client is told here. if let Some(kicker) = &kicker { - kicker.start_attempts().forget_branch(&branch_id); + if kicker.start_attempts().forget_branch(&branch_id) { + crate::workspace_setup::failures::notify_queue_released(&app_handle, &branch_id); + } } return Ok(false); }; @@ -12737,12 +12741,13 @@ mod tests { /// A held row stays `queued` with no error, and its reason used to go to /// the log alone. The user is told why it waits as a notice on the /// branch's setup event, once per cause and not on every recheck. A - /// holder that is `disconnected` is named for the reconcile service once - /// per cause too: a recheck comes every few seconds, and asking on each - /// would skip the spacing that service keeps between its Blox calls. + /// holder that is `disconnected` is named for the reconcile service on + /// every look, whichever session of the sibling it is: that service + /// spaces the attempts (`Trigger::QueueHold`), so the row's wait is + /// bounded by that spacing and not by a failure backoff. /// When nothing holds the row any more the notice is taken back, once. #[test] - fn a_held_queue_says_why_and_has_a_disconnected_holder_checked_once_per_cause() { + fn a_held_queue_says_why_once_per_cause_and_names_a_disconnected_holder_on_every_look() { use crate::session_queue::BranchQueueKicker; use tauri::Listener; @@ -12793,7 +12798,7 @@ mod tests { assert_eq!(row.error_message, None); // The holder loses its connection: a new cause, told once, and the - // session is named for the reconcile service on that look alone. + // session is named for the reconcile service on every look. store .update_session_status(&agent.id, store::SessionStatus::Disconnected, None, None) .unwrap(); @@ -12811,13 +12816,13 @@ mod tests { look(), SiblingHold { announce: None, - disconnected_holder: None, + disconnected_holder: Some(agent.id.clone()), } ); } - // Back to running and lost again: each is a new cause, and the - // second loss is asked about once more. + // Back to running: nobody to ask about. Lost again: asked about + // again. store .update_session_status(&agent.id, store::SessionStatus::Running, None, None) .unwrap(); @@ -12829,7 +12834,28 @@ mod tests { look().disconnected_holder.as_deref(), Some(agent.id.as_str()) ); - assert_eq!(look().disconnected_holder, None); + + // That session settles and another `disconnected` session of the + // same sibling branch has the clone. The reason reads the same, so + // the user is told nothing new, and the new holder is still named. + let other = store::Session::new_running("more", Path::new("/tmp")).with_branch(&sibling.id); + store.create_session(&other).unwrap(); + store + .update_session_status(&other.id, store::SessionStatus::Disconnected, None, None) + .unwrap(); + store + .update_session_status(&agent.id, store::SessionStatus::Completed, None, None) + .unwrap(); + assert_eq!( + look(), + SiblingHold { + announce: None, + disconnected_holder: Some(other.id.clone()), + } + ); + store + .update_session_status(&other.id, store::SessionStatus::Completed, None, None) + .unwrap(); // Settled: nothing holds the row, nobody is asked about, and the // notice is taken back with the same heading and no text, once. diff --git a/apps/staged/src-tauri/src/session_queue/retry.rs b/apps/staged/src-tauri/src/session_queue/retry.rs index af502e69e..84996857c 100644 --- a/apps/staged/src-tauri/src/session_queue/retry.rs +++ b/apps/staged/src-tauri/src/session_queue/retry.rs @@ -59,10 +59,11 @@ pub(crate) fn queued_start_retry(failures: u32) -> QueuedStartRetry { } /// How long a held row waits before the drain looks at what holds it again. -/// A recheck is a database read and nothing more, so it is short: the queue -/// moves within seconds of the cause going away. What costs a Blox call, -/// asking the reconcile service about a `disconnected` holder, is done once -/// per cause and not on each recheck. +/// A recheck is a database read, so it is short: the queue moves within +/// seconds of the cause going away. It also tells the reconcile service +/// that the queue waits on a `disconnected` holder, which costs a Blox call +/// only when that service lets the ask through: once per +/// `reconcile::QUEUE_HOLD_MIN_SPACING` for the holder, not on each recheck. pub(crate) const QUEUE_HOLD_RECHECK: Duration = Duration::from_secs(5); struct Attempt { @@ -195,9 +196,14 @@ impl QueuedStartAttempts { dropped_held } - /// Drop every entry for a branch that is gone. - pub(crate) fn forget_branch(&self, branch_id: &str) { - self.by_branch.lock().unwrap().remove(branch_id); + /// Drop every entry for a branch that is gone. Returns whether a held + /// row was among them. + pub(crate) fn forget_branch(&self, branch_id: &str) -> bool { + self.by_branch + .lock() + .unwrap() + .remove(branch_id) + .is_some_and(|rows| rows.values().any(|attempt| attempt.held.is_some())) } #[cfg(test)] diff --git a/apps/staged/src-tauri/src/session_queue/tests.rs b/apps/staged/src-tauri/src/session_queue/tests.rs index d6cfd30fc..4fe73a482 100644 --- a/apps/staged/src-tauri/src/session_queue/tests.rs +++ b/apps/staged/src-tauri/src/session_queue/tests.rs @@ -341,8 +341,14 @@ fn queued_start_attempts_retry_with_growing_delays_then_give_up() { assert!(attempts.retain_queued("b", |_| false)); assert!(attempts.is_empty()); + // A branch that is gone takes its rows with it, and says whether one + // of them was held, for the same reason. attempts.record_failure("gone", "s3", t2); - attempts.forget_branch("gone"); + assert!(!attempts.forget_branch("gone")); + attempts.record_failure("gone", "s3", t2); + attempts.hold("gone", "s4", t2, "waiting"); + assert!(attempts.forget_branch("gone")); + assert!(!attempts.forget_branch("gone")); assert!(attempts.is_empty()); } diff --git a/apps/staged/src-tauri/src/session_runner/reconcile.rs b/apps/staged/src-tauri/src/session_runner/reconcile.rs index 869a29f34..102678966 100644 --- a/apps/staged/src-tauri/src/session_runner/reconcile.rs +++ b/apps/staged/src-tauri/src/session_runner/reconcile.rs @@ -28,8 +28,10 @@ //! //! Reconcile runs on launch ([`reconcile_all`]), from a sweep every //! [`SWEEP_INTERVAL`] that respects each row's backoff, after a successful -//! Blox call for the row's workstation ([`note_workspace_reachable`]), and on -//! request ([`reconcile_session`], for a "Check now"). Every write is a +//! Blox call for the row's workstation ([`note_workspace_reachable`]), while +//! a queued session of another branch waits on the row +//! ([`note_queue_held_behind`]), and on request ([`reconcile_session`], for a +//! "Check now"). Every write is a //! compare-and-swap on the row as it was read, so a Stop, or another Staged //! instance's reconcile, that got there first wins, and running the same //! snapshot twice changes nothing the second time. @@ -73,6 +75,16 @@ const BACKOFF_CAP: Duration = Duration::from_secs(30 * 60); /// The least time between two attempts a successful Blox call can set off for /// one row: a burst of `ws exec`s (a diff load) should not attach once each. const NUDGE_MIN_SPACING: Duration = Duration::from_secs(30); +/// The least time between two attempts for a row that a queued session of +/// another branch waits on ([`Trigger::QueueHold`]). The queue asks every few +/// seconds for as long as it is held; this is what bounds that to one attempt +/// a minute per row, and the queue's wait after the cause has cleared to this +/// plus one of its rechecks. One minute is [`BACKOFF_BASE`] and +/// [`HELD_WORK_RECHECK`]: the spacing this service keeps by itself after a +/// row's first failed attempt, so a held queue never has a row tried more +/// often than that. It only stops the spacing growing, to +/// [`BACKOFF_CAP`], while something waits on the answer. +pub(crate) const QUEUE_HOLD_MIN_SPACING: Duration = Duration::from_secs(60); /// The wait before checking again on a row whose session is still working /// after its turn; it doubles with each further check that finds the work /// still going, up to [`HELD_WORK_RECHECK_CAP`]. Not a failure backoff: the @@ -165,6 +177,9 @@ pub(crate) enum Trigger { /// A Blox call for the row's workstation just succeeded: rows that have /// failed before, spaced by [`NUDGE_MIN_SPACING`]. WorkspaceReachable, + /// A queued session of another branch waits on the row: regardless of + /// backoff, spaced by [`QUEUE_HOLD_MIN_SPACING`]. + QueueHold, /// The user asked ("Check now"): regardless of backoff. CheckNow, } @@ -410,6 +425,9 @@ impl Reconciler { .last_attempt .map_or(true, |last| now.duration_since(last) >= NUDGE_MIN_SPACING) } + Trigger::QueueHold => row.last_attempt.map_or(true, |last| { + now.duration_since(last) >= QUEUE_HOLD_MIN_SPACING + }), }; if !allowed { return false; @@ -708,6 +726,17 @@ pub(crate) fn note_workspace_reachable(workspace_name: &str) { } } +/// A queued session of another branch cannot start until `session_id`, which +/// is `disconnected`, has settled. Called on every look the queue takes at +/// its hold, which is every few seconds: an attempt starts only when the +/// row's last one began at least [`QUEUE_HOLD_MIN_SPACING`] ago, whatever its +/// backoff says. A map lookup otherwise. +pub(crate) fn note_queue_held_behind(session_id: &str) { + if let Some(reconciler) = active() { + reconciler.spawn_reconcile(session_id, Trigger::QueueHold); + } +} + /// Re-run the sweep for the life of the process, always on the current /// reconciler. fn spawn_sweep() { diff --git a/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs b/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs index 233d2a53d..926a294f7 100644 --- a/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs +++ b/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs @@ -2702,6 +2702,60 @@ fn a_reachable_workstation_retries_only_its_failed_rows() { .reconcile_blocking(&failed.id, Trigger::WorkspaceReachable)); } +/// A queue held behind a row asks about it every few seconds. The row is +/// tried at once when this process never has, and after that once per +/// [`QUEUE_HOLD_MIN_SPACING`] whatever its failures have backed off to, so the +/// asks cost a bounded number of Blox calls and the queue waits no longer +/// than that spacing once the cause has cleared. +#[test] +fn a_held_queue_has_its_holder_tried_once_per_spacing_whatever_the_backoff() { + let fx = Fixture::new(); + let holder = fx.disconnected_commit_session(); + fx.blox.set_workspace(WorkspaceProbe::Unreachable { + kind: UnreachableKind::Unreachable, + message: "timed out".to_string(), + }); + let ask = || { + fx.reconciler + .reconcile_blocking(&holder.id, Trigger::QueueHold) + }; + let age_last_attempt = |age: Duration| { + fx.reconciler + .rows + .lock() + .unwrap() + .get_mut(&holder.id) + .unwrap() + .last_attempt = Some(std::time::Instant::now() - age); + }; + + // Never tried: the first ask is an attempt, the next ones are not. + assert!(ask()); + let calls = fx.blox.calls().len(); + for _ in 0..5 { + assert!(!ask()); + } + age_last_attempt(QUEUE_HOLD_MIN_SPACING - Duration::from_secs(5)); + assert!(!ask()); + assert_eq!(fx.blox.calls().len(), calls); + + // Failures have pushed the sweep's next attempt out to the cap. The + // hold does not wait for it: one attempt per spacing, and no more. + for _ in 0..6 { + age_last_attempt(QUEUE_HOLD_MIN_SPACING); + assert!(ask()); + assert!(!ask()); + } + assert!(!fx.reconciler.reconcile_blocking(&holder.id, Trigger::Sweep)); + assert_eq!(fx.reconciler.states()[0].attempts, 7); + + // The cause clears: the ask after the spacing settles the row. + fx.blox.set_workspace(WorkspaceProbe::Running); + age_last_attempt(QUEUE_HOLD_MIN_SPACING); + assert!(ask()); + assert!(fx.reconciler.states().is_empty()); +} + #[test] fn states_report_rows_being_reconciled() { let fx = Fixture::new(); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index 39594160f..b31fe8c81 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -608,14 +608,26 @@ fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { "kicker.start_attempts().is_held(&branch_id, &session.id)\n && hold_queue_behind_sibling_session(", "start_queued_session_for_branch(", ); - // The hold tells the user why the row waits and has a `disconnected` - // holder looked at, once per cause, and takes the notice back when the - // row is no longer held: both drain call sites go through the one - // function. A held row that left the queue has it taken back too. - assert_calls( - "drain_branch_queue", - drain, - "crate::workspace_setup::failures::notify_queue_released(&app_handle, &branch_id)", + // The hold tells the user why the row waits, once per cause, names a + // `disconnected` holder for the reconcile service with the hold's own + // spaced trigger, and takes the notice back when the row is no longer + // held: both drain call sites go through the one function. A held row + // that left the queue has it taken back too, and so has one whose + // branch is gone. + for released in [ + "if kicker.start_attempts().forget_branch(&branch_id) {", + ".retain_queued(&branch_id, |id| queued.iter().any(|s| s.id == id))", + ] { + assert_calls("drain_branch_queue", drain, released); + } + assert_eq!( + drain + .matches( + "crate::workspace_setup::failures::notify_queue_released(&app_handle, &branch_id)" + ) + .count(), + 2, + "one take-back for a branch that is gone, one for a row that left the queue" ); let hold = body_of(SESSION_COMMANDS, "fn hold_queue_behind_sibling_session<"); for call in [ @@ -623,7 +635,7 @@ fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { "crate::workspace_setup::failures::notify_queue_held(app_handle, &branch.id, notice)", "kicker.start_attempts().release(&branch.id, session_id)", "crate::workspace_setup::failures::notify_queue_released(app_handle, &branch.id)", - "crate::session_runner::reconcile::reconcile_session(holder.clone())", + "crate::session_runner::reconcile::note_queue_held_behind(holder)", ] { assert_calls("hold_queue_behind_sibling_session", hold, call); } diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts index 70b6ffc71..166de2793 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.svelte.ts @@ -29,6 +29,11 @@ interface WorkspaceSetupErrorEvent { noticeTitle?: string; } +/** The heading the backend sends with why a branch's queued session is + * waiting, and alone when it no longer is (`QUEUE_HELD_TITLE` in + * `workspace_setup/failures.rs`). */ +const QUEUE_HELD_TITLE = 'Queued session is waiting'; + /** The one toast a branch has for why its queued session is waiting: a new * cause replaces the old one, and it is taken down when the wait is over. */ function queueHeldToastId(branchId: string): string { @@ -364,7 +369,7 @@ class WorkspaceLifecycleController { this.setupErrorEventSeqByBranch.set(branchId, ++this.setupErrorEventSeq); if (error == null) { this.clearWorkspaceError(branchId); - if (event.noticeTitle) { + if (event.noticeTitle === QUEUE_HELD_TITLE) { // Why the branch's queued session waits: up for as long as it does. const id = queueHeldToastId(branchId); if (event.notice) { @@ -373,7 +378,8 @@ class WorkspaceLifecycleController { toast.dismiss(id); } } else if (event.notice) { - toast.warning('Workspace branch was given existing commits', { + // Any other heading is a notice of its own, not the hold's. + toast.warning(event.noticeTitle ?? 'Workspace branch was given existing commits', { description: event.notice, duration: Infinity, }); diff --git a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts index 7b14f5c5a..dd0e4ee5b 100644 --- a/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts +++ b/apps/staged/src/lib/features/projects/workspaceLifecycle.test.ts @@ -541,6 +541,33 @@ describe('WorkspaceLifecycleController.retryWorktree', () => { lifecycle.stop(); }); + // Only the hold's own heading is the hold notice. Another heading is a + // notice of its own: it does not take the hold toast's id, so it neither + // replaces that toast nor is taken down with it, and alone it does nothing. + it('does not treat a notice under another heading as the queue hold', async () => { + const lifecycle = await newController(); + await settle(); + + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + notice: 'Something else happened.', + noticeTitle: 'Something else', + }); + expect(toastWarning.mock.calls).toEqual([ + ['Something else', { description: 'Something else happened.', duration: Infinity }], + ]); + + emit('workspace-setup-error', { + branchId: 'branch-1', + error: null, + noticeTitle: 'Something else', + }); + expect(toastWarning).toHaveBeenCalledTimes(1); + expect(toastDismiss).not.toHaveBeenCalled(); + lifecycle.stop(); + }); + // The backend reports every failed run, and a snapshot covers the ones // from before the window mounted. it('keeps setup reasons from the error event and the snapshot taken at start', async () => { From 6f5a055fc413f4b1a4b2b621b50e2d0afac3d7e4 Mon Sep 17 00:00:00 2001 From: Matt Toohey <contact@matttoohey.com> Date: Thu, 19 Nov 2026 12:59:49 +1100 Subject: [PATCH 9/9] fix(staged): keep a held queue's attempts out of the reconcile backoff and pin the hold heading on both sides Applies the warning and the suggestion of the review of 85b01e61, and nothing else. Backoff. A queued row held behind a sibling branch's `disconnected` session has the holder tried once a minute, whatever its backoff says. Each of those attempts that failed counted as a failure, so about six minutes of hold over a failing holder left the service's next attempt thirty minutes out. When the hold then ended with the holder still disconnected (the queued row stopped or force-started, its branch deleted or out of Running), the sweep left that session alone for up to half an hour. The session's reconcile state also showed that next attempt and a failure count growing by one a minute. An attempt the hold asked for before the row's `next_attempt` was due is now marked on the row when it begins (`RowState::ahead_of_schedule`, in memory only). When it ends in a backoff, `failures` and `next_attempt` stay as they were, so the service's own schedule after the hold is what it would have been without the hold. The reason and the phase are still recorded, and the log line gives the time left to the standing next attempt. A hold attempt at or after the time the service would have tried anyway is that attempt and counts as usual, as does the first attempt on a row never tried. The held-work path is unchanged. The comment on `QUEUE_HOLD_MIN_SPACING` said the trigger only stops the spacing growing. It now says the hold's attempts come on top of the service's schedule and leave it alone. Heading. The frontend tells the hold notice from any other by its heading, which is a string literal on each side. The hold's wiring test now asserts that `workspaceLifecycle.svelte.ts` contains `failures::QUEUE_HELD_TITLE`, so a reworded Rust heading fails a test instead of sending the notice down the other branch. Tests: the reconcile test that pinned seven attempts with the sweep refused now pins one counted failure and an unmoved next attempt after six hold attempts, and that an ask once the service's attempt is due counts as usual. No data model changes and no event changes. Signed-off-by: Matt Toohey <contact@matttoohey.com> Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Signed-off-by: Matt Toohey <contact@matttoohey.com> --- .../src-tauri/src/session_runner/reconcile.rs | 30 +++++++++++++---- .../src/session_runner/reconcile/tests.rs | 33 ++++++++++++++++--- .../src/workspace_setup/tests/wiring.rs | 4 +++ 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/apps/staged/src-tauri/src/session_runner/reconcile.rs b/apps/staged/src-tauri/src/session_runner/reconcile.rs index 102678966..0457b281b 100644 --- a/apps/staged/src-tauri/src/session_runner/reconcile.rs +++ b/apps/staged/src-tauri/src/session_runner/reconcile.rs @@ -82,8 +82,11 @@ const NUDGE_MIN_SPACING: Duration = Duration::from_secs(30); /// plus one of its rechecks. One minute is [`BACKOFF_BASE`] and /// [`HELD_WORK_RECHECK`]: the spacing this service keeps by itself after a /// row's first failed attempt, so a held queue never has a row tried more -/// often than that. It only stops the spacing growing, to -/// [`BACKOFF_CAP`], while something waits on the answer. +/// often than that. These attempts come on top of the service's own schedule +/// and leave it alone: one that began before the row's next attempt was due +/// and failed counts no failure and moves no next attempt +/// (`RowState::ahead_of_schedule`), so once the hold is over the row is tried +/// when it would have been without the hold. pub(crate) const QUEUE_HOLD_MIN_SPACING: Duration = Duration::from_secs(60); /// The wait before checking again on a row whose session is still working /// after its turn; it doubles with each further check that finds the work @@ -200,6 +203,10 @@ struct RowState { last_attempt: Option<Instant>, next_attempt: Option<Instant>, next_attempt_at: Option<i64>, + /// The attempt in flight is one a held queue asked for + /// ([`Trigger::QueueHold`]) before `next_attempt` was due: if it fails, + /// `failures` and `next_attempt` stay as they are. + ahead_of_schedule: bool, /// How long the session has been working after its turn, as attempts /// found it ([`RowOutcome::Held`]), for [`Reconciler::held_work_ran_out`]. held: Option<HeldClock>, @@ -434,6 +441,8 @@ impl Reconciler { } row.in_flight = true; row.last_attempt = Some(now); + row.ahead_of_schedule = + trigger == Trigger::QueueHold && row.next_attempt.is_some_and(|next| next > now); true } @@ -534,10 +543,19 @@ impl Reconciler { .map(|row| forgotten(session_id, row)), RowOutcome::Backoff(reason) => rows.get_mut(session_id).map(|row| { row.in_flight = false; - row.failures += 1; - let delay = backoff(row.failures); - row.next_attempt = Some(Instant::now() + delay); - row.next_attempt_at = Some(now_timestamp() + delay.as_millis() as i64); + let now = Instant::now(); + let delay = match row.next_attempt { + // A held queue's attempt, made before this one was + // due: the service's own schedule stands. + Some(next) if row.ahead_of_schedule => next.saturating_duration_since(now), + _ => { + row.failures += 1; + let delay = backoff(row.failures); + row.next_attempt = Some(now + delay); + row.next_attempt_at = Some(now_timestamp() + delay.as_millis() as i64); + delay + } + }; log::info!( "[reconcile] Session {session_id} stays disconnected ({:?}: {}); \ retrying in {}s", diff --git a/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs b/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs index 926a294f7..f53e53813 100644 --- a/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs +++ b/apps/staged/src-tauri/src/session_runner/reconcile/tests.rs @@ -2706,7 +2706,9 @@ fn a_reachable_workstation_retries_only_its_failed_rows() { /// tried at once when this process never has, and after that once per /// [`QUEUE_HOLD_MIN_SPACING`] whatever its failures have backed off to, so the /// asks cost a bounded number of Blox calls and the queue waits no longer -/// than that spacing once the cause has cleared. +/// than that spacing once the cause has cleared. An attempt made before the +/// service's own next one was due leaves that schedule as it was, so the row +/// is not backed off further for having been waited on. #[test] fn a_held_queue_has_its_holder_tried_once_per_spacing_whatever_the_backoff() { let fx = Fixture::new(); @@ -2729,8 +2731,11 @@ fn a_held_queue_has_its_holder_tried_once_per_spacing_whatever_the_backoff() { .last_attempt = Some(std::time::Instant::now() - age); }; - // Never tried: the first ask is an attempt, the next ones are not. + // Never tried: the first ask is an attempt, the next ones are not. The + // service would have made that attempt itself, so it counts as a failure. assert!(ask()); + let scheduled = fx.reconciler.states()[0].next_attempt_at; + assert!(scheduled.is_some()); let calls = fx.blox.calls().len(); for _ in 0..5 { assert!(!ask()); @@ -2739,15 +2744,33 @@ fn a_held_queue_has_its_holder_tried_once_per_spacing_whatever_the_backoff() { assert!(!ask()); assert_eq!(fx.blox.calls().len(), calls); - // Failures have pushed the sweep's next attempt out to the cap. The - // hold does not wait for it: one attempt per spacing, and no more. + // The hold does not wait for the sweep's next attempt: one attempt per + // spacing, and no more. Those fail ahead of the service's schedule and + // leave it as it was: no failure counted, the sweep due when it was. for _ in 0..6 { age_last_attempt(QUEUE_HOLD_MIN_SPACING); assert!(ask()); assert!(!ask()); } assert!(!fx.reconciler.reconcile_blocking(&holder.id, Trigger::Sweep)); - assert_eq!(fx.reconciler.states()[0].attempts, 7); + let state = fx.reconciler.states().remove(0); + assert_eq!(state.attempts, 1); + assert_eq!(state.next_attempt_at, scheduled); + + // An ask once the service's own attempt is due is that attempt, and + // counts as usual. + fx.reconciler + .rows + .lock() + .unwrap() + .get_mut(&holder.id) + .unwrap() + .next_attempt = Some(std::time::Instant::now()); + age_last_attempt(QUEUE_HOLD_MIN_SPACING); + assert!(ask()); + let state = fx.reconciler.states().remove(0); + assert_eq!(state.attempts, 2); + assert!(state.next_attempt_at > scheduled); // The cause clears: the ask after the spacing settles the row. fx.blox.set_workspace(WorkspaceProbe::Running); diff --git a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs index b31fe8c81..e0becf8f1 100644 --- a/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs +++ b/apps/staged/src-tauri/src/workspace_setup/tests/wiring.rs @@ -25,6 +25,8 @@ const PR_CREATE: &str = include_str!("../../prs/create_pr.rs"); const PR_PULL: &str = include_str!("../../prs/pull.rs"); const TRANSFER: &str = include_str!("../../project_transfer/orchestrator.rs"); const TRANSFER_COMMANDS: &str = include_str!("../../project_transfer/commands.rs"); +const FRONTEND_LIFECYCLE: &str = + include_str!("../../../../src/lib/features/projects/workspaceLifecycle.svelte.ts"); /// The text of the function whose signature starts with `signature`, to the /// closing brace at its own indentation. @@ -645,4 +647,6 @@ fn the_queue_is_held_not_drained_behind_a_start_that_could_not_check_out() { look, "crate::workspace_setup::clone::sibling_session_in_clone(store, branch)", ); + // The frontend tells the hold notice from any other by its heading. + assert!(FRONTEND_LIFECYCLE.contains(crate::workspace_setup::failures::QUEUE_HELD_TITLE)); }