Skip to content

[codex] Add agent funnel telemetry #262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
samuelstroschein merged 1 commit into from
Jul 2, 2026
Merged

[codex] Add agent funnel telemetry #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions electron/types.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,14 @@ export type DesktopAppApi = {

export type DesktopTelemetryEventName =
| "agent_opened"
| "agent_start_failed"
| "app_opened"
| "diff_opened"
| "diff_resolved"
| "document_open_attempted"
| "document_modified"
| "document_viewed"
| "prompt_submitted"
| "workspace_recovery_lifecycle"
| "workspace_extension_profiled"
| "workspace_profiled";
Expand Down
213 changes: 213 additions & 0 deletions src/extensions/terminal/index.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ const xtermMock = vi.hoisted(() => ({
dispose: ReturnType<typeof vi.fn>;
}>,
}));
const telemetryMock = vi.hoisted(() => ({
captureTelemetry: vi.fn(),
captureTelemetryException: vi.fn(),
}));

vi.mock("@xterm/xterm", () => {
class MockTerminal {
Expand All @@ -35,6 +39,8 @@ vi.mock("@xterm/addon-fit", () => {
return { FitAddon: MockFitAddon };
});

vi.mock("@/lib/telemetry", () => telemetryMock);

import { TerminalView } from "./index";

const originalDesktop = window.flashtypeDesktop;
Expand All @@ -43,6 +49,8 @@ const originalResizeObserver = window.ResizeObserver;
describe("TerminalView", () => {
beforeEach(() => {
xtermMock.instances.length = 0;
telemetryMock.captureTelemetry.mockClear();
telemetryMock.captureTelemetryException.mockClear();
window.ResizeObserver = class {
observe = vi.fn();
disconnect = vi.fn();
Expand Down Expand Up @@ -93,6 +101,18 @@ describe("TerminalView", () => {
screen.getByText(/Flashtype needs Claude Code 2\.1\.78 or newer/u),
).toBeInTheDocument();
expect(xtermMock.instances[0]?.writeln).not.toHaveBeenCalled();
expect(telemetryMock.captureTelemetry).toHaveBeenCalledWith(
"agent_start_failed",
{
agent: "claude",
reason: "unsupported",
required_version: "2.1.78",
detected_version: "2.1.77",
surface: "terminal",
retry_count: 0,
},
);
expect(telemetryMock.captureTelemetryException).not.toHaveBeenCalled();

fireEvent.click(screen.getByRole("button", { name: /retry/i }));

Expand All @@ -104,8 +124,201 @@ describe("TerminalView", () => {
}),
);
});

test.each([
["missing", "Claude Code not found"],
["unparseable", "Could not read Claude Code version"],
["timeout", "Claude Code version check timed out"],
["failed", "Claude Code version check failed"],
] as const)(
"captures agent version startup telemetry for %s errors",
async (reason, title) => {
const terminalApi = createTerminalApi();
terminalApi.create = vi.fn().mockResolvedValueOnce({
status: "agentVersionError",
agent: "claude",
requiredVersion: "2.1.78",
reason,
});
window.flashtypeDesktop = createDesktop(terminalApi);

render(
<TerminalView
launchConfig={{
initialCommand: "claude-flashtype",
pathWrapper: {
executableName: "claude-flashtype",
command: "claude --settings '{}'",
},
}}
/>,
);

expect(await screen.findByText(title)).toBeInTheDocument();
expect(telemetryMock.captureTelemetry).toHaveBeenCalledWith(
"agent_start_failed",
{
agent: "claude",
reason,
required_version: "2.1.78",
detected_version: undefined,
surface: "terminal",
retry_count: 0,
},
);
expect(telemetryMock.captureTelemetryException).not.toHaveBeenCalled();
},
);

test("captures retry count on later startup failures", async () => {
const terminalApi = createTerminalApi();
terminalApi.create = vi
.fn()
.mockResolvedValueOnce({
status: "agentVersionError",
agent: "codex",
requiredVersion: "0.134.0",
detectedVersion: "0.133.0",
reason: "unsupported",
})
.mockResolvedValueOnce({
status: "agentVersionError",
agent: "codex",
requiredVersion: "0.134.0",
detectedVersion: "0.133.0",
reason: "unsupported",
});
window.flashtypeDesktop = createDesktop(terminalApi);

render(
<TerminalView
launchConfig={{
initialCommand: "codex-flashtype",
pathWrapper: {
executableName: "codex-flashtype",
command: "codex --dangerously-bypass-hook-trust",
},
}}
/>,
);

expect(
await screen.findByText("Codex update required"),
).toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: /retry/i }));

await waitFor(() => expect(terminalApi.create).toHaveBeenCalledTimes(2));
await waitFor(() =>
expect(telemetryMock.captureTelemetry).toHaveBeenLastCalledWith(
"agent_start_failed",
{
agent: "codex",
reason: "unsupported",
required_version: "0.134.0",
detected_version: "0.133.0",
surface: "terminal",
retry_count: 1,
},
),
);
});

test("captures unexpected agent startup errors as telemetry and exceptions", async () => {
const terminalApi = createTerminalApi();
const error = new Error("pty failed");
terminalApi.create = vi.fn().mockRejectedValueOnce(error);
window.flashtypeDesktop = createDesktop(terminalApi);

render(
<TerminalView
launchConfig={{
initialCommand: "codex-flashtype",
pathWrapper: {
executableName: "codex-flashtype",
command: "codex --dangerously-bypass-hook-trust",
},
}}
/>,
);

expect(
await screen.findByText("Failed to start terminal"),
).toBeInTheDocument();
expect(telemetryMock.captureTelemetry).toHaveBeenCalledWith(
"agent_start_failed",
{
agent: "codex",
reason: "unexpected",
surface: "terminal",
retry_count: 0,
},
);
expect(telemetryMock.captureTelemetryException).toHaveBeenCalledWith(
error,
{
agent: "codex",
reason: "unexpected",
surface: "terminal",
retry_count: 0,
},
);
});

test("captures initial command write failures as agent startup telemetry", async () => {
const terminalApi = createTerminalApi();
const error = new Error("write failed");
terminalApi.create = vi.fn().mockResolvedValueOnce({
status: "ok",
id: "terminal-1",
});
terminalApi.write = vi.fn().mockRejectedValueOnce(error);
window.flashtypeDesktop = createDesktop(terminalApi);

render(
<TerminalView
launchConfig={{
initialCommand: "claude-flashtype",
pathWrapper: {
executableName: "claude-flashtype",
command: "claude --settings '{}'",
},
}}
/>,
);

expect(
await screen.findByText("Failed to start terminal"),
).toBeInTheDocument();
expect(telemetryMock.captureTelemetry).toHaveBeenCalledWith(
"agent_start_failed",
{
agent: "claude",
reason: "unexpected",
surface: "terminal",
retry_count: 0,
},
);
expect(telemetryMock.captureTelemetryException).toHaveBeenCalledWith(
error,
{
agent: "claude",
reason: "unexpected",
surface: "terminal",
retry_count: 0,
},
);
});
});

function createDesktop(terminalApi: DesktopTerminalApi) {
return {
lix: {
workspaceDir: vi.fn().mockResolvedValue("/workspace"),
},
terminal: terminalApi,
} as unknown as Window["flashtypeDesktop"];
}

function createTerminalApi(): DesktopTerminalApi {
return {
create: vi.fn(),
Expand Down
66 changes: 63 additions & 3 deletions src/extensions/terminal/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { FitAddon } from "@xterm/addon-fit";
import { Terminal } from "@xterm/xterm";
import "@xterm/xterm/css/xterm.css";
import { Button } from "@/components/ui/button";
import { captureTelemetry, captureTelemetryException } from "@/lib/telemetry";
import { createReactExtensionDefinition } from "../../extension-runtime/react-extension";
import { TERMINAL_EXTENSION_KIND } from "../../extension-runtime/extension-instance-helpers";
import {
Expand Down Expand Up @@ -57,6 +58,7 @@ type AgentVersionErrorResult = Extract<
type TerminalStartupError =
| { kind: "agentVersion"; error: AgentVersionErrorResult }
| { kind: "unexpected"; error: unknown };
type AgentTelemetryName = AgentVersionErrorResult["agent"];

export function TerminalView({
launchConfig,
Expand Down Expand Up @@ -84,6 +86,7 @@ export function TerminalView({
let disposed = false;
let cleanedUp = false;
let terminalId: string | null = null;
const attemptLaunchConfig = launchConfigRef.current;

setStartupError(null);
const terminal = new Terminal({
Expand Down Expand Up @@ -156,6 +159,10 @@ export function TerminalView({

const showStartupError = (error: TerminalStartupError) => {
if (disposed) return;
captureAgentStartFailure(error, {
launchConfig: attemptLaunchConfig,
retryCount: retryKey,
});
setStartupError(error);
cleanupLocalTerminal();
};
Expand All @@ -170,7 +177,7 @@ export function TerminalView({
cwd,
cols: terminal.cols,
rows: terminal.rows,
pathWrapper: launchConfigRef.current.pathWrapper,
pathWrapper: attemptLaunchConfig.pathWrapper,
});
if (created.status === "agentVersionError") {
showStartupError({ kind: "agentVersion", error: created });
Expand All @@ -182,11 +189,11 @@ export function TerminalView({
}
terminalId = created.id;
handleResize();
const command = launchConfigRef.current.initialCommand;
launchConfigRef.current = {};
const command = attemptLaunchConfig.initialCommand;
if (command) {
await terminalApi.write({ id: created.id, data: `${command}\r` });
}
launchConfigRef.current = {};
} catch (error) {
showStartupError({ kind: "unexpected", error });
}
Expand Down Expand Up @@ -227,6 +234,59 @@ export function TerminalView({
);
}

function captureAgentStartFailure(
error: TerminalStartupError,
options: {
readonly launchConfig: TerminalLaunchConfig;
readonly retryCount: number;
},
) {
const properties = agentStartFailureTelemetryProperties(error, options);
if (!properties) return;
captureTelemetry("agent_start_failed", properties);
if (error.kind === "unexpected") {
captureTelemetryException(error.error, properties);
}
}

function agentStartFailureTelemetryProperties(
error: TerminalStartupError,
options: {
readonly launchConfig: TerminalLaunchConfig;
readonly retryCount: number;
},
) {
const common = {
surface: "terminal",
retry_count: options.retryCount,
} as const;
if (error.kind === "agentVersion") {
return {
...common,
agent: error.error.agent,
reason: error.error.reason,
required_version: error.error.requiredVersion,
detected_version: error.error.detectedVersion,
};
}
const agent = readAgentTelemetryName(options.launchConfig);
if (!agent) return null;
return {
...common,
agent,
reason: "unexpected",
};
}

function readAgentTelemetryName(
launchConfig: TerminalLaunchConfig,
): AgentTelemetryName | null {
const executableName = launchConfig.pathWrapper?.executableName;
if (executableName === "claude-flashtype") return "claude";
if (executableName === "codex-flashtype") return "codex";
return null;
}

function TerminalStartupErrorView({
error,
onRetry,
Expand Down
Loading
Loading