From fb16eab4fb5ec3844a7cf4942c04316806189bac Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 13:48:34 -0700 Subject: [PATCH 01/84] Back out @lib indirection in tsconfig.json --- codex-cli/package-lock.json | 4 +- codex-cli/src/app.tsx | 2 +- codex-cli/src/{lib => }/approvals.ts | 2 +- codex-cli/src/cli.tsx | 4 +- .../src/components/chat/multiline-editor.tsx | 2 +- .../chat/terminal-chat-tool-call-item.tsx | 2 +- .../src/components/chat/terminal-chat.tsx | 4 +- codex-cli/src/{lib => }/format-command.ts | 0 codex-cli/src/{lib => }/parse-apply-patch.ts | 0 codex-cli/src/{lib => }/text-buffer.ts | 0 codex-cli/src/utils/agent/agent-loop.ts | 2 +- codex-cli/src/utils/agent/exec.ts | 2 +- .../src/utils/agent/handle-exec-command.ts | 6 +- codex-cli/src/utils/agent/review.ts | 2 +- codex-cli/src/utils/parsers.ts | 55 ++----------------- codex-cli/tests/agent-cancel-early.test.ts | 4 +- .../tests/agent-cancel-prev-response.test.ts | 4 +- codex-cli/tests/agent-cancel-race.test.ts | 4 +- codex-cli/tests/agent-cancel.test.ts | 4 +- .../tests/agent-function-call-id.test.ts | 4 +- .../tests/agent-generic-network-error.test.ts | 4 +- .../tests/agent-invalid-request-error.test.ts | 4 +- .../tests/agent-max-tokens-error.test.ts | 4 +- codex-cli/tests/agent-network-errors.test.ts | 4 +- codex-cli/tests/agent-project-doc.test.ts | 4 +- .../tests/agent-rate-limit-error.test.ts | 4 +- codex-cli/tests/agent-server-retry.test.ts | 4 +- codex-cli/tests/agent-terminate.test.ts | 4 +- codex-cli/tests/agent-thinking-time.test.ts | 4 +- codex-cli/tests/approvals.test.ts | 4 +- codex-cli/tests/external-editor.test.ts | 2 +- codex-cli/tests/format-command.test.ts | 2 +- .../tests/invalid-command-handling.test.ts | 4 +- ...ultiline-external-editor-shortcut.test.tsx | 2 +- .../tests/multiline-history-behavior.test.tsx | 6 +- codex-cli/tests/parse-apply-patch.test.ts | 2 +- .../tests/text-buffer-copy-paste.test.ts | 2 +- codex-cli/tests/text-buffer-crlf.test.ts | 2 +- codex-cli/tests/text-buffer-gaps.test.ts | 2 +- codex-cli/tests/text-buffer-word.test.ts | 2 +- codex-cli/tests/text-buffer.test.ts | 2 +- codex-cli/tsconfig.json | 3 - 42 files changed, 65 insertions(+), 113 deletions(-) rename codex-cli/src/{lib => }/approvals.ts (99%) rename codex-cli/src/{lib => }/format-command.ts (100%) rename codex-cli/src/{lib => }/parse-apply-patch.ts (100%) rename codex-cli/src/{lib => }/text-buffer.ts (100%) diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json index d29561b53a..589ee59659 100644 --- a/codex-cli/package-lock.json +++ b/codex-cli/package-lock.json @@ -1,12 +1,12 @@ { "name": "@openai/codex", - "version": "0.1.04160940", + "version": "0.1.04161241", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@openai/codex", - "version": "0.1.04160940", + "version": "0.1.04161241", "license": "Apache-2.0", "dependencies": { "@inkjs/ui": "^2.0.0", diff --git a/codex-cli/src/app.tsx b/codex-cli/src/app.tsx index dbb0cdedc8..c0b8c6f4e3 100644 --- a/codex-cli/src/app.tsx +++ b/codex-cli/src/app.tsx @@ -1,5 +1,5 @@ +import type { ApprovalPolicy } from "./approvals"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import TerminalChat from "./components/chat/terminal-chat"; diff --git a/codex-cli/src/lib/approvals.ts b/codex-cli/src/approvals.ts similarity index 99% rename from codex-cli/src/lib/approvals.ts rename to codex-cli/src/approvals.ts index 8985939a43..0cf3703b54 100644 --- a/codex-cli/src/lib/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -3,7 +3,7 @@ import type { ParseEntry, ControlOperator } from "shell-quote"; import { identify_files_added, identify_files_needed, -} from "../utils/agent/apply-patch"; +} from "./utils/agent/apply-patch"; import * as path from "path"; import { parse } from "shell-quote"; diff --git a/codex-cli/src/cli.tsx b/codex-cli/src/cli.tsx index 5e43dff1f3..0af421a431 100644 --- a/codex-cli/src/cli.tsx +++ b/codex-cli/src/cli.tsx @@ -1,9 +1,9 @@ #!/usr/bin/env node import type { AppRollout } from "./app"; +import type { ApprovalPolicy } from "./approvals"; import type { CommandConfirmation } from "./utils/agent/agent-loop"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import App from "./app"; @@ -124,7 +124,7 @@ const cli = meow( fullContext: { type: "boolean", aliases: ["f"], - description: `Run in full-context editing approach. The model is given the whole code + description: `Run in full-context editing approach. The model is given the whole code directory as context and performs changes in one go without acting.`, }, }, diff --git a/codex-cli/src/components/chat/multiline-editor.tsx b/codex-cli/src/components/chat/multiline-editor.tsx index c18555447e..c99961bbcd 100644 --- a/codex-cli/src/components/chat/multiline-editor.tsx +++ b/codex-cli/src/components/chat/multiline-editor.tsx @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { useTerminalSize } from "../../hooks/use-terminal-size"; -import TextBuffer from "../../lib/text-buffer.js"; +import TextBuffer from "../../text-buffer.js"; import chalk from "chalk"; import { Box, Text, useInput, useStdin } from "ink"; import { EventEmitter } from "node:events"; diff --git a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx index 1aeb7d7e98..5853460884 100644 --- a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx +++ b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx @@ -1,5 +1,5 @@ +import { parseApplyPatch } from "../../parse-apply-patch"; import { shortenPath } from "../../utils/short-path"; -import { parseApplyPatch } from "@lib/parse-apply-patch"; import chalk from "chalk"; import { Text } from "ink"; import React from "react"; diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index bbc9bb052a..35fecec564 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -1,6 +1,6 @@ +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { CommandConfirmation } from "../../utils/agent/agent-loop.js"; import type { AppConfig } from "../../utils/config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ColorName } from "chalk"; import type { ResponseItem } from "openai/resources/responses/responses.mjs"; import type { ReviewDecision } from "src/utils/agent/review.ts"; @@ -12,6 +12,7 @@ import { uniqueById, } from "./terminal-chat-utils.js"; import TerminalMessageHistory from "./terminal-message-history.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; @@ -25,7 +26,6 @@ import ApprovalModeOverlay from "../approval-mode-overlay.js"; import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; import { Box, Text } from "ink"; import React, { useEffect, useMemo, useState } from "react"; import { inspect } from "util"; diff --git a/codex-cli/src/lib/format-command.ts b/codex-cli/src/format-command.ts similarity index 100% rename from codex-cli/src/lib/format-command.ts rename to codex-cli/src/format-command.ts diff --git a/codex-cli/src/lib/parse-apply-patch.ts b/codex-cli/src/parse-apply-patch.ts similarity index 100% rename from codex-cli/src/lib/parse-apply-patch.ts rename to codex-cli/src/parse-apply-patch.ts diff --git a/codex-cli/src/lib/text-buffer.ts b/codex-cli/src/text-buffer.ts similarity index 100% rename from codex-cli/src/lib/text-buffer.ts rename to codex-cli/src/text-buffer.ts diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index 65127fd85c..d0db6f125d 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -1,6 +1,6 @@ import type { ReviewDecision } from "./review.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { AppConfig } from "../config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ResponseFunctionToolCall, ResponseInputItem, diff --git a/codex-cli/src/utils/agent/exec.ts b/codex-cli/src/utils/agent/exec.ts index aade68a860..a441f192f8 100644 --- a/codex-cli/src/utils/agent/exec.ts +++ b/codex-cli/src/utils/agent/exec.ts @@ -5,7 +5,7 @@ import { process_patch } from "./apply-patch.js"; import { SandboxType } from "./sandbox/interface.js"; import { execWithSeatbelt } from "./sandbox/macos-seatbelt.js"; import { exec as rawExec } from "./sandbox/raw-exec.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import fs from "fs"; import os from "os"; diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 5eaa11d660..41b7abbcd4 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -1,7 +1,7 @@ import type { CommandConfirmation } from "./agent-loop.js"; import type { AppConfig } from "../config.js"; import type { ExecInput } from "./sandbox/interface.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { ResponseInputItem } from "openai/resources/responses/responses.mjs"; import { exec, execApplyPatch } from "./exec.js"; @@ -9,8 +9,8 @@ import { isLoggingEnabled, log } from "./log.js"; import { ReviewDecision } from "./review.js"; import { FullAutoErrorMode } from "../auto-approval-mode.js"; import { SandboxType } from "./sandbox/interface.js"; -import { canAutoApprove } from "@lib/approvals.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { canAutoApprove } from "../../approvals.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { access } from "fs/promises"; // --------------------------------------------------------------------------- diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index ed2af0ce17..a370388569 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,4 +1,4 @@ -import type { SafeCommandReason } from "@lib/approvals"; +import type { SafeCommandReason } from "../../approvals"; export type CommandReviewDetails = { cmd: Array; diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index cb477c1d7d..815e7b2f00 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -3,11 +3,13 @@ import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; -import type { SafeCommandReason } from "@lib/approvals.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; +import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; import process from "process"; +import { parse } from "shell-quote"; +import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is @@ -23,52 +25,6 @@ const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ ";", ]); -// Lazily resolve heavy dependencies at runtime to avoid test environments -// (which might not have the @lib alias configured) from failing at import -// time. If the modules cannot be loaded we fall back to permissive stub -// implementations so that basic functionality – like unit‑testing small UI -// helpers – continues to work without the full codex‑lib dependency tree. - -let isSafeCommand: (cmd: Array) => SafeCommandReason | null = () => - null; -let shellQuoteParse: - | ((cmd: string, env?: Record) => Array) - | undefined; -let formatCommandForDisplay: (cmd: Array) => string = (cmd) => - cmd.join(" "); - -async function loadLibs(): Promise { - try { - const approvals = await import("@lib/approvals.js"); - if (typeof approvals.isSafeCommand === "function") { - isSafeCommand = approvals.isSafeCommand; - } - } catch { - // ignore – keep stub - } - try { - const fmt = await import("@lib/format-command.js"); - if (typeof fmt.formatCommandForDisplay === "function") { - formatCommandForDisplay = fmt.formatCommandForDisplay; - } - } catch { - // ignore – keep stub - } - try { - const sq = await import("shell-quote"); - if (typeof sq.parse === "function") { - shellQuoteParse = sq.parse as typeof shellQuoteParse; - } - } catch { - // ignore – keep stub - } -} - -// Trigger the dynamic import in the background; callers that need the real -// implementation should await the returned promise (parsers currently does not -// require this for correctness during tests). -void loadLibs(); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -175,10 +131,9 @@ function computeAutoApproval(cmd: Array): SafeCommandReason | null { cmd.length === 3 && cmd[0] === "bash" && cmd[1] === "-lc" && - typeof cmd[2] === "string" && - shellQuoteParse + typeof cmd[2] === "string" ) { - const parsed = shellQuoteParse(cmd[2], process.env ?? {}); + const parsed = parse(cmd[2], process.env ?? {}); if (parsed.length === 0) { return null; } diff --git a/codex-cli/tests/agent-cancel-early.test.ts b/codex-cli/tests/agent-cancel-early.test.ts index b7b5fdcae7..b235a6d6bc 100644 --- a/codex-cli/tests/agent-cancel-early.test.ts +++ b/codex-cli/tests/agent-cancel-early.test.ts @@ -64,13 +64,13 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-prev-response.test.ts b/codex-cli/tests/agent-cancel-prev-response.test.ts index 4047f88452..fe73c338cc 100644 --- a/codex-cli/tests/agent-cancel-prev-response.test.ts +++ b/codex-cli/tests/agent-cancel-prev-response.test.ts @@ -71,13 +71,13 @@ vi.mock("openai", () => { }); // Stub helpers not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-race.test.ts b/codex-cli/tests/agent-cancel-race.test.ts index c9c1845d8e..89e7cca744 100644 --- a/codex-cli/tests/agent-cancel-race.test.ts +++ b/codex-cli/tests/agent-cancel-race.test.ts @@ -67,11 +67,11 @@ vi.mock("openai", () => { }); // Stubs for external helpers referenced indirectly. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel.test.ts b/codex-cli/tests/agent-cancel.test.ts index 93ff5ba317..69c17f7f7e 100644 --- a/codex-cli/tests/agent-cancel.test.ts +++ b/codex-cli/tests/agent-cancel.test.ts @@ -47,7 +47,7 @@ vi.mock("openai", () => { }); // Mock the approvals and formatCommand helpers referenced by handle‑exec‑command. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -57,7 +57,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-function-call-id.test.ts b/codex-cli/tests/agent-function-call-id.test.ts index 7d0a55d122..d50c08eea4 100644 --- a/codex-cli/tests/agent-function-call-id.test.ts +++ b/codex-cli/tests/agent-function-call-id.test.ts @@ -88,14 +88,14 @@ vi.mock("openai", () => { }); // Stub approvals & command formatting – not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-generic-network-error.test.ts b/codex-cli/tests/agent-generic-network-error.test.ts index 8c636fbda3..942adff668 100644 --- a/codex-cli/tests/agent-generic-network-error.test.ts +++ b/codex-cli/tests/agent-generic-network-error.test.ts @@ -23,14 +23,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – unrelated to network handling. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-invalid-request-error.test.ts b/codex-cli/tests/agent-invalid-request-error.test.ts index 631d451944..090d0b52d9 100644 --- a/codex-cli/tests/agent-invalid-request-error.test.ts +++ b/codex-cli/tests/agent-invalid-request-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-max-tokens-error.test.ts b/codex-cli/tests/agent-max-tokens-error.test.ts index 5e01229454..de4fd17026 100644 --- a/codex-cli/tests/agent-max-tokens-error.test.ts +++ b/codex-cli/tests/agent-max-tokens-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-network-errors.test.ts b/codex-cli/tests/agent-network-errors.test.ts index e01b08e918..f98ea5bf19 100644 --- a/codex-cli/tests/agent-network-errors.test.ts +++ b/codex-cli/tests/agent-network-errors.test.ts @@ -42,14 +42,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – not relevant here. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-project-doc.test.ts b/codex-cli/tests/agent-project-doc.test.ts index 97ad9c8c34..d3050f3953 100644 --- a/codex-cli/tests/agent-project-doc.test.ts +++ b/codex-cli/tests/agent-project-doc.test.ts @@ -51,7 +51,7 @@ vi.mock("openai", () => { // The AgentLoop pulls these helpers in order to decide whether a command can // be auto‑approved. None of that matters for this test, so we stub the module // with minimal no‑op implementations. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -61,7 +61,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-rate-limit-error.test.ts b/codex-cli/tests/agent-rate-limit-error.test.ts index 318b52f348..abe91171c4 100644 --- a/codex-cli/tests/agent-rate-limit-error.test.ts +++ b/codex-cli/tests/agent-rate-limit-error.test.ts @@ -52,14 +52,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – not relevant to rate‑limit handling. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-server-retry.test.ts b/codex-cli/tests/agent-server-retry.test.ts index df98e69e95..e851dd9778 100644 --- a/codex-cli/tests/agent-server-retry.test.ts +++ b/codex-cli/tests/agent-server-retry.test.ts @@ -32,14 +32,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-terminate.test.ts b/codex-cli/tests/agent-terminate.test.ts index 7a60456518..bce77437af 100644 --- a/codex-cli/tests/agent-terminate.test.ts +++ b/codex-cli/tests/agent-terminate.test.ts @@ -49,7 +49,7 @@ vi.mock("openai", () => { // --- Helpers referenced by handle‑exec‑command ----------------------------- -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -59,7 +59,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-thinking-time.test.ts b/codex-cli/tests/agent-thinking-time.test.ts index c94d8a5e5e..7132070084 100644 --- a/codex-cli/tests/agent-thinking-time.test.ts +++ b/codex-cli/tests/agent-thinking-time.test.ts @@ -74,12 +74,12 @@ vi.mock("openai", () => { }); // Stub helpers referenced indirectly so we do not pull in real FS/network -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index e06db94092..7cb0bd3d3e 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -1,6 +1,6 @@ -import type { SafetyAssessment } from "../src/lib/approvals"; +import type { SafetyAssessment } from "../src/approvals"; -import { canAutoApprove } from "../src/lib/approvals"; +import { canAutoApprove } from "../src/approvals"; import { describe, test, expect } from "vitest"; describe("canAutoApprove()", () => { diff --git a/codex-cli/tests/external-editor.test.ts b/codex-cli/tests/external-editor.test.ts index d530be5ecc..77041c2870 100644 --- a/codex-cli/tests/external-editor.test.ts +++ b/codex-cli/tests/external-editor.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect, vi } from "vitest"; /* ------------------------------------------------------------------------- diff --git a/codex-cli/tests/format-command.test.ts b/codex-cli/tests/format-command.test.ts index 3981d9b106..7de417308a 100644 --- a/codex-cli/tests/format-command.test.ts +++ b/codex-cli/tests/format-command.test.ts @@ -1,4 +1,4 @@ -import { formatCommandForDisplay } from "../src/lib/format-command"; +import { formatCommandForDisplay } from "../src/format-command"; import { describe, test, expect } from "vitest"; describe("formatCommandForDisplay()", () => { diff --git a/codex-cli/tests/invalid-command-handling.test.ts b/codex-cli/tests/invalid-command-handling.test.ts index 6619b4f240..a3f87a7251 100644 --- a/codex-cli/tests/invalid-command-handling.test.ts +++ b/codex-cli/tests/invalid-command-handling.test.ts @@ -22,7 +22,7 @@ describe("rawExec – invalid command handling", () => { // --------------------------------------------------------------------------- // Mock approvals and logging helpers so the test focuses on execution flow. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, canAutoApprove: () => @@ -31,7 +31,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx index 158a5e655d..9b2e2f25e5 100644 --- a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx +++ b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx @@ -1,6 +1,6 @@ import { renderTui } from "./ui-test-helpers.js"; import MultilineTextEditor from "../src/components/chat/multiline-editor.js"; -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import * as React from "react"; import { describe, it, expect, vi } from "vitest"; diff --git a/codex-cli/tests/multiline-history-behavior.test.tsx b/codex-cli/tests/multiline-history-behavior.test.tsx index 5c906837ee..cada52ddab 100644 --- a/codex-cli/tests/multiline-history-behavior.test.tsx +++ b/codex-cli/tests/multiline-history-behavior.test.tsx @@ -34,12 +34,12 @@ vi.mock("../src/utils/input-utils.js", () => ({ })), })); -// Mock the optional @lib/* dependencies so the dynamic import in parsers.ts +// Mock the optional ../src/* dependencies so the dynamic import in parsers.ts // does not fail during the test environment where the alias isn't configured. -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ formatCommandForDisplay: (cmd: Array) => cmd.join(" "), })); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ isSafeCommand: (_cmd: Array) => null, })); diff --git a/codex-cli/tests/parse-apply-patch.test.ts b/codex-cli/tests/parse-apply-patch.test.ts index 0195542e56..53aa119093 100644 --- a/codex-cli/tests/parse-apply-patch.test.ts +++ b/codex-cli/tests/parse-apply-patch.test.ts @@ -1,4 +1,4 @@ -import { parseApplyPatch } from "../src/lib/parse-apply-patch"; +import { parseApplyPatch } from "../src/parse-apply-patch"; import { expect, test, describe } from "vitest"; // Helper function to unwrap a non‑null result in tests that expect success. diff --git a/codex-cli/tests/text-buffer-copy-paste.test.ts b/codex-cli/tests/text-buffer-copy-paste.test.ts index 311b2b9aa6..cc1fd119e5 100644 --- a/codex-cli/tests/text-buffer-copy-paste.test.ts +++ b/codex-cli/tests/text-buffer-copy-paste.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; // These tests ensure that the TextBuffer copy‑&‑paste logic keeps parity with diff --git a/codex-cli/tests/text-buffer-crlf.test.ts b/codex-cli/tests/text-buffer-crlf.test.ts index 4b33b498b5..736c22a27d 100644 --- a/codex-cli/tests/text-buffer-crlf.test.ts +++ b/codex-cli/tests/text-buffer-crlf.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; describe("TextBuffer – newline normalisation", () => { diff --git a/codex-cli/tests/text-buffer-gaps.test.ts b/codex-cli/tests/text-buffer-gaps.test.ts index 986ad37eed..046d468e8a 100644 --- a/codex-cli/tests/text-buffer-gaps.test.ts +++ b/codex-cli/tests/text-buffer-gaps.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; // The purpose of this test‑suite is NOT to make the implementation green today diff --git a/codex-cli/tests/text-buffer-word.test.ts b/codex-cli/tests/text-buffer-word.test.ts index 009786a28c..4ea7679450 100644 --- a/codex-cli/tests/text-buffer-word.test.ts +++ b/codex-cli/tests/text-buffer-word.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, test, expect } from "vitest"; describe("TextBuffer – word‑wise navigation & deletion", () => { diff --git a/codex-cli/tests/text-buffer.test.ts b/codex-cli/tests/text-buffer.test.ts index ae78f29409..c3f33d0fa1 100644 --- a/codex-cli/tests/text-buffer.test.ts +++ b/codex-cli/tests/text-buffer.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; describe("TextBuffer – basic editing parity with Rust suite", () => { diff --git a/codex-cli/tsconfig.json b/codex-cli/tsconfig.json index 626fc5dbf4..d1dacc9149 100644 --- a/codex-cli/tsconfig.json +++ b/codex-cli/tsconfig.json @@ -11,9 +11,6 @@ ], "types": ["node"], "baseUrl": "./", - "paths": { - "@lib/*": ["./src/lib/*"] - }, "resolveJsonModule": false, // ESM doesn't yet support JSON modules. "jsx": "react", "declaration": true, From 66904fc9256815c2603f4c06e772f3f978b1b730 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 13:51:46 -0700 Subject: [PATCH 02/84] Back out @lib indirection in tsconfig.json --- codex-cli/src/app.tsx | 2 +- codex-cli/src/{lib => }/approvals.ts | 2 +- codex-cli/src/cli.tsx | 4 +- .../src/components/chat/multiline-editor.tsx | 2 +- .../chat/terminal-chat-tool-call-item.tsx | 2 +- .../src/components/chat/terminal-chat.tsx | 4 +- codex-cli/src/{lib => }/format-command.ts | 0 codex-cli/src/{lib => }/parse-apply-patch.ts | 0 codex-cli/src/{lib => }/text-buffer.ts | 0 codex-cli/src/utils/agent/agent-loop.ts | 2 +- codex-cli/src/utils/agent/exec.ts | 2 +- .../src/utils/agent/handle-exec-command.ts | 6 +- codex-cli/src/utils/agent/review.ts | 2 +- codex-cli/src/utils/parsers.ts | 55 ++----------------- codex-cli/tests/agent-cancel-early.test.ts | 4 +- .../tests/agent-cancel-prev-response.test.ts | 4 +- codex-cli/tests/agent-cancel-race.test.ts | 4 +- codex-cli/tests/agent-cancel.test.ts | 4 +- .../tests/agent-function-call-id.test.ts | 4 +- .../tests/agent-generic-network-error.test.ts | 4 +- .../tests/agent-invalid-request-error.test.ts | 4 +- .../tests/agent-max-tokens-error.test.ts | 4 +- codex-cli/tests/agent-network-errors.test.ts | 4 +- codex-cli/tests/agent-project-doc.test.ts | 4 +- .../tests/agent-rate-limit-error.test.ts | 4 +- codex-cli/tests/agent-server-retry.test.ts | 4 +- codex-cli/tests/agent-terminate.test.ts | 4 +- codex-cli/tests/agent-thinking-time.test.ts | 4 +- codex-cli/tests/approvals.test.ts | 4 +- codex-cli/tests/external-editor.test.ts | 2 +- codex-cli/tests/format-command.test.ts | 2 +- .../tests/invalid-command-handling.test.ts | 4 +- ...ultiline-external-editor-shortcut.test.tsx | 2 +- .../tests/multiline-history-behavior.test.tsx | 6 +- codex-cli/tests/parse-apply-patch.test.ts | 2 +- .../tests/text-buffer-copy-paste.test.ts | 2 +- codex-cli/tests/text-buffer-crlf.test.ts | 2 +- codex-cli/tests/text-buffer-gaps.test.ts | 2 +- codex-cli/tests/text-buffer-word.test.ts | 2 +- codex-cli/tests/text-buffer.test.ts | 2 +- codex-cli/tsconfig.json | 3 - 41 files changed, 63 insertions(+), 111 deletions(-) rename codex-cli/src/{lib => }/approvals.ts (99%) rename codex-cli/src/{lib => }/format-command.ts (100%) rename codex-cli/src/{lib => }/parse-apply-patch.ts (100%) rename codex-cli/src/{lib => }/text-buffer.ts (100%) diff --git a/codex-cli/src/app.tsx b/codex-cli/src/app.tsx index dbb0cdedc8..c0b8c6f4e3 100644 --- a/codex-cli/src/app.tsx +++ b/codex-cli/src/app.tsx @@ -1,5 +1,5 @@ +import type { ApprovalPolicy } from "./approvals"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import TerminalChat from "./components/chat/terminal-chat"; diff --git a/codex-cli/src/lib/approvals.ts b/codex-cli/src/approvals.ts similarity index 99% rename from codex-cli/src/lib/approvals.ts rename to codex-cli/src/approvals.ts index 8985939a43..0cf3703b54 100644 --- a/codex-cli/src/lib/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -3,7 +3,7 @@ import type { ParseEntry, ControlOperator } from "shell-quote"; import { identify_files_added, identify_files_needed, -} from "../utils/agent/apply-patch"; +} from "./utils/agent/apply-patch"; import * as path from "path"; import { parse } from "shell-quote"; diff --git a/codex-cli/src/cli.tsx b/codex-cli/src/cli.tsx index 5e43dff1f3..0af421a431 100644 --- a/codex-cli/src/cli.tsx +++ b/codex-cli/src/cli.tsx @@ -1,9 +1,9 @@ #!/usr/bin/env node import type { AppRollout } from "./app"; +import type { ApprovalPolicy } from "./approvals"; import type { CommandConfirmation } from "./utils/agent/agent-loop"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import App from "./app"; @@ -124,7 +124,7 @@ const cli = meow( fullContext: { type: "boolean", aliases: ["f"], - description: `Run in full-context editing approach. The model is given the whole code + description: `Run in full-context editing approach. The model is given the whole code directory as context and performs changes in one go without acting.`, }, }, diff --git a/codex-cli/src/components/chat/multiline-editor.tsx b/codex-cli/src/components/chat/multiline-editor.tsx index c18555447e..c99961bbcd 100644 --- a/codex-cli/src/components/chat/multiline-editor.tsx +++ b/codex-cli/src/components/chat/multiline-editor.tsx @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { useTerminalSize } from "../../hooks/use-terminal-size"; -import TextBuffer from "../../lib/text-buffer.js"; +import TextBuffer from "../../text-buffer.js"; import chalk from "chalk"; import { Box, Text, useInput, useStdin } from "ink"; import { EventEmitter } from "node:events"; diff --git a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx index 1aeb7d7e98..5853460884 100644 --- a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx +++ b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx @@ -1,5 +1,5 @@ +import { parseApplyPatch } from "../../parse-apply-patch"; import { shortenPath } from "../../utils/short-path"; -import { parseApplyPatch } from "@lib/parse-apply-patch"; import chalk from "chalk"; import { Text } from "ink"; import React from "react"; diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index bbc9bb052a..35fecec564 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -1,6 +1,6 @@ +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { CommandConfirmation } from "../../utils/agent/agent-loop.js"; import type { AppConfig } from "../../utils/config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ColorName } from "chalk"; import type { ResponseItem } from "openai/resources/responses/responses.mjs"; import type { ReviewDecision } from "src/utils/agent/review.ts"; @@ -12,6 +12,7 @@ import { uniqueById, } from "./terminal-chat-utils.js"; import TerminalMessageHistory from "./terminal-message-history.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; @@ -25,7 +26,6 @@ import ApprovalModeOverlay from "../approval-mode-overlay.js"; import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; import { Box, Text } from "ink"; import React, { useEffect, useMemo, useState } from "react"; import { inspect } from "util"; diff --git a/codex-cli/src/lib/format-command.ts b/codex-cli/src/format-command.ts similarity index 100% rename from codex-cli/src/lib/format-command.ts rename to codex-cli/src/format-command.ts diff --git a/codex-cli/src/lib/parse-apply-patch.ts b/codex-cli/src/parse-apply-patch.ts similarity index 100% rename from codex-cli/src/lib/parse-apply-patch.ts rename to codex-cli/src/parse-apply-patch.ts diff --git a/codex-cli/src/lib/text-buffer.ts b/codex-cli/src/text-buffer.ts similarity index 100% rename from codex-cli/src/lib/text-buffer.ts rename to codex-cli/src/text-buffer.ts diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index e2b465733d..d88604c471 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -1,6 +1,6 @@ import type { ReviewDecision } from "./review.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { AppConfig } from "../config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ResponseFunctionToolCall, ResponseInputItem, diff --git a/codex-cli/src/utils/agent/exec.ts b/codex-cli/src/utils/agent/exec.ts index aade68a860..a441f192f8 100644 --- a/codex-cli/src/utils/agent/exec.ts +++ b/codex-cli/src/utils/agent/exec.ts @@ -5,7 +5,7 @@ import { process_patch } from "./apply-patch.js"; import { SandboxType } from "./sandbox/interface.js"; import { execWithSeatbelt } from "./sandbox/macos-seatbelt.js"; import { exec as rawExec } from "./sandbox/raw-exec.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import fs from "fs"; import os from "os"; diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 5eaa11d660..41b7abbcd4 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -1,7 +1,7 @@ import type { CommandConfirmation } from "./agent-loop.js"; import type { AppConfig } from "../config.js"; import type { ExecInput } from "./sandbox/interface.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { ResponseInputItem } from "openai/resources/responses/responses.mjs"; import { exec, execApplyPatch } from "./exec.js"; @@ -9,8 +9,8 @@ import { isLoggingEnabled, log } from "./log.js"; import { ReviewDecision } from "./review.js"; import { FullAutoErrorMode } from "../auto-approval-mode.js"; import { SandboxType } from "./sandbox/interface.js"; -import { canAutoApprove } from "@lib/approvals.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { canAutoApprove } from "../../approvals.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { access } from "fs/promises"; // --------------------------------------------------------------------------- diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index ed2af0ce17..a370388569 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,4 +1,4 @@ -import type { SafeCommandReason } from "@lib/approvals"; +import type { SafeCommandReason } from "../../approvals"; export type CommandReviewDetails = { cmd: Array; diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index cb477c1d7d..815e7b2f00 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -3,11 +3,13 @@ import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; -import type { SafeCommandReason } from "@lib/approvals.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; +import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; import process from "process"; +import { parse } from "shell-quote"; +import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is @@ -23,52 +25,6 @@ const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ ";", ]); -// Lazily resolve heavy dependencies at runtime to avoid test environments -// (which might not have the @lib alias configured) from failing at import -// time. If the modules cannot be loaded we fall back to permissive stub -// implementations so that basic functionality – like unit‑testing small UI -// helpers – continues to work without the full codex‑lib dependency tree. - -let isSafeCommand: (cmd: Array) => SafeCommandReason | null = () => - null; -let shellQuoteParse: - | ((cmd: string, env?: Record) => Array) - | undefined; -let formatCommandForDisplay: (cmd: Array) => string = (cmd) => - cmd.join(" "); - -async function loadLibs(): Promise { - try { - const approvals = await import("@lib/approvals.js"); - if (typeof approvals.isSafeCommand === "function") { - isSafeCommand = approvals.isSafeCommand; - } - } catch { - // ignore – keep stub - } - try { - const fmt = await import("@lib/format-command.js"); - if (typeof fmt.formatCommandForDisplay === "function") { - formatCommandForDisplay = fmt.formatCommandForDisplay; - } - } catch { - // ignore – keep stub - } - try { - const sq = await import("shell-quote"); - if (typeof sq.parse === "function") { - shellQuoteParse = sq.parse as typeof shellQuoteParse; - } - } catch { - // ignore – keep stub - } -} - -// Trigger the dynamic import in the background; callers that need the real -// implementation should await the returned promise (parsers currently does not -// require this for correctness during tests). -void loadLibs(); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -175,10 +131,9 @@ function computeAutoApproval(cmd: Array): SafeCommandReason | null { cmd.length === 3 && cmd[0] === "bash" && cmd[1] === "-lc" && - typeof cmd[2] === "string" && - shellQuoteParse + typeof cmd[2] === "string" ) { - const parsed = shellQuoteParse(cmd[2], process.env ?? {}); + const parsed = parse(cmd[2], process.env ?? {}); if (parsed.length === 0) { return null; } diff --git a/codex-cli/tests/agent-cancel-early.test.ts b/codex-cli/tests/agent-cancel-early.test.ts index b7b5fdcae7..b235a6d6bc 100644 --- a/codex-cli/tests/agent-cancel-early.test.ts +++ b/codex-cli/tests/agent-cancel-early.test.ts @@ -64,13 +64,13 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-prev-response.test.ts b/codex-cli/tests/agent-cancel-prev-response.test.ts index 4047f88452..fe73c338cc 100644 --- a/codex-cli/tests/agent-cancel-prev-response.test.ts +++ b/codex-cli/tests/agent-cancel-prev-response.test.ts @@ -71,13 +71,13 @@ vi.mock("openai", () => { }); // Stub helpers not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-race.test.ts b/codex-cli/tests/agent-cancel-race.test.ts index c9c1845d8e..89e7cca744 100644 --- a/codex-cli/tests/agent-cancel-race.test.ts +++ b/codex-cli/tests/agent-cancel-race.test.ts @@ -67,11 +67,11 @@ vi.mock("openai", () => { }); // Stubs for external helpers referenced indirectly. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel.test.ts b/codex-cli/tests/agent-cancel.test.ts index 93ff5ba317..69c17f7f7e 100644 --- a/codex-cli/tests/agent-cancel.test.ts +++ b/codex-cli/tests/agent-cancel.test.ts @@ -47,7 +47,7 @@ vi.mock("openai", () => { }); // Mock the approvals and formatCommand helpers referenced by handle‑exec‑command. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -57,7 +57,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-function-call-id.test.ts b/codex-cli/tests/agent-function-call-id.test.ts index 7d0a55d122..d50c08eea4 100644 --- a/codex-cli/tests/agent-function-call-id.test.ts +++ b/codex-cli/tests/agent-function-call-id.test.ts @@ -88,14 +88,14 @@ vi.mock("openai", () => { }); // Stub approvals & command formatting – not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-generic-network-error.test.ts b/codex-cli/tests/agent-generic-network-error.test.ts index 8c636fbda3..942adff668 100644 --- a/codex-cli/tests/agent-generic-network-error.test.ts +++ b/codex-cli/tests/agent-generic-network-error.test.ts @@ -23,14 +23,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – unrelated to network handling. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-invalid-request-error.test.ts b/codex-cli/tests/agent-invalid-request-error.test.ts index 631d451944..090d0b52d9 100644 --- a/codex-cli/tests/agent-invalid-request-error.test.ts +++ b/codex-cli/tests/agent-invalid-request-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-max-tokens-error.test.ts b/codex-cli/tests/agent-max-tokens-error.test.ts index 5e01229454..de4fd17026 100644 --- a/codex-cli/tests/agent-max-tokens-error.test.ts +++ b/codex-cli/tests/agent-max-tokens-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-network-errors.test.ts b/codex-cli/tests/agent-network-errors.test.ts index e01b08e918..f98ea5bf19 100644 --- a/codex-cli/tests/agent-network-errors.test.ts +++ b/codex-cli/tests/agent-network-errors.test.ts @@ -42,14 +42,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – not relevant here. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-project-doc.test.ts b/codex-cli/tests/agent-project-doc.test.ts index 97ad9c8c34..d3050f3953 100644 --- a/codex-cli/tests/agent-project-doc.test.ts +++ b/codex-cli/tests/agent-project-doc.test.ts @@ -51,7 +51,7 @@ vi.mock("openai", () => { // The AgentLoop pulls these helpers in order to decide whether a command can // be auto‑approved. None of that matters for this test, so we stub the module // with minimal no‑op implementations. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -61,7 +61,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-rate-limit-error.test.ts b/codex-cli/tests/agent-rate-limit-error.test.ts index 18779450aa..9782744679 100644 --- a/codex-cli/tests/agent-rate-limit-error.test.ts +++ b/codex-cli/tests/agent-rate-limit-error.test.ts @@ -34,14 +34,14 @@ vi.mock("openai", () => { // Stub helpers that the agent indirectly imports so it does not attempt any // file‑system access or real approvals logic during the test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-server-retry.test.ts b/codex-cli/tests/agent-server-retry.test.ts index 9ec4eb5aac..09278f2ceb 100644 --- a/codex-cli/tests/agent-server-retry.test.ts +++ b/codex-cli/tests/agent-server-retry.test.ts @@ -32,14 +32,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-terminate.test.ts b/codex-cli/tests/agent-terminate.test.ts index 7a60456518..bce77437af 100644 --- a/codex-cli/tests/agent-terminate.test.ts +++ b/codex-cli/tests/agent-terminate.test.ts @@ -49,7 +49,7 @@ vi.mock("openai", () => { // --- Helpers referenced by handle‑exec‑command ----------------------------- -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -59,7 +59,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-thinking-time.test.ts b/codex-cli/tests/agent-thinking-time.test.ts index c94d8a5e5e..7132070084 100644 --- a/codex-cli/tests/agent-thinking-time.test.ts +++ b/codex-cli/tests/agent-thinking-time.test.ts @@ -74,12 +74,12 @@ vi.mock("openai", () => { }); // Stub helpers referenced indirectly so we do not pull in real FS/network -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index e06db94092..7cb0bd3d3e 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -1,6 +1,6 @@ -import type { SafetyAssessment } from "../src/lib/approvals"; +import type { SafetyAssessment } from "../src/approvals"; -import { canAutoApprove } from "../src/lib/approvals"; +import { canAutoApprove } from "../src/approvals"; import { describe, test, expect } from "vitest"; describe("canAutoApprove()", () => { diff --git a/codex-cli/tests/external-editor.test.ts b/codex-cli/tests/external-editor.test.ts index d530be5ecc..77041c2870 100644 --- a/codex-cli/tests/external-editor.test.ts +++ b/codex-cli/tests/external-editor.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect, vi } from "vitest"; /* ------------------------------------------------------------------------- diff --git a/codex-cli/tests/format-command.test.ts b/codex-cli/tests/format-command.test.ts index 3981d9b106..7de417308a 100644 --- a/codex-cli/tests/format-command.test.ts +++ b/codex-cli/tests/format-command.test.ts @@ -1,4 +1,4 @@ -import { formatCommandForDisplay } from "../src/lib/format-command"; +import { formatCommandForDisplay } from "../src/format-command"; import { describe, test, expect } from "vitest"; describe("formatCommandForDisplay()", () => { diff --git a/codex-cli/tests/invalid-command-handling.test.ts b/codex-cli/tests/invalid-command-handling.test.ts index 6619b4f240..a3f87a7251 100644 --- a/codex-cli/tests/invalid-command-handling.test.ts +++ b/codex-cli/tests/invalid-command-handling.test.ts @@ -22,7 +22,7 @@ describe("rawExec – invalid command handling", () => { // --------------------------------------------------------------------------- // Mock approvals and logging helpers so the test focuses on execution flow. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, canAutoApprove: () => @@ -31,7 +31,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx index 158a5e655d..9b2e2f25e5 100644 --- a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx +++ b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx @@ -1,6 +1,6 @@ import { renderTui } from "./ui-test-helpers.js"; import MultilineTextEditor from "../src/components/chat/multiline-editor.js"; -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import * as React from "react"; import { describe, it, expect, vi } from "vitest"; diff --git a/codex-cli/tests/multiline-history-behavior.test.tsx b/codex-cli/tests/multiline-history-behavior.test.tsx index 5c906837ee..cada52ddab 100644 --- a/codex-cli/tests/multiline-history-behavior.test.tsx +++ b/codex-cli/tests/multiline-history-behavior.test.tsx @@ -34,12 +34,12 @@ vi.mock("../src/utils/input-utils.js", () => ({ })), })); -// Mock the optional @lib/* dependencies so the dynamic import in parsers.ts +// Mock the optional ../src/* dependencies so the dynamic import in parsers.ts // does not fail during the test environment where the alias isn't configured. -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ formatCommandForDisplay: (cmd: Array) => cmd.join(" "), })); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ isSafeCommand: (_cmd: Array) => null, })); diff --git a/codex-cli/tests/parse-apply-patch.test.ts b/codex-cli/tests/parse-apply-patch.test.ts index 0195542e56..53aa119093 100644 --- a/codex-cli/tests/parse-apply-patch.test.ts +++ b/codex-cli/tests/parse-apply-patch.test.ts @@ -1,4 +1,4 @@ -import { parseApplyPatch } from "../src/lib/parse-apply-patch"; +import { parseApplyPatch } from "../src/parse-apply-patch"; import { expect, test, describe } from "vitest"; // Helper function to unwrap a non‑null result in tests that expect success. diff --git a/codex-cli/tests/text-buffer-copy-paste.test.ts b/codex-cli/tests/text-buffer-copy-paste.test.ts index 311b2b9aa6..cc1fd119e5 100644 --- a/codex-cli/tests/text-buffer-copy-paste.test.ts +++ b/codex-cli/tests/text-buffer-copy-paste.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; // These tests ensure that the TextBuffer copy‑&‑paste logic keeps parity with diff --git a/codex-cli/tests/text-buffer-crlf.test.ts b/codex-cli/tests/text-buffer-crlf.test.ts index 4b33b498b5..736c22a27d 100644 --- a/codex-cli/tests/text-buffer-crlf.test.ts +++ b/codex-cli/tests/text-buffer-crlf.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; describe("TextBuffer – newline normalisation", () => { diff --git a/codex-cli/tests/text-buffer-gaps.test.ts b/codex-cli/tests/text-buffer-gaps.test.ts index 986ad37eed..046d468e8a 100644 --- a/codex-cli/tests/text-buffer-gaps.test.ts +++ b/codex-cli/tests/text-buffer-gaps.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; // The purpose of this test‑suite is NOT to make the implementation green today diff --git a/codex-cli/tests/text-buffer-word.test.ts b/codex-cli/tests/text-buffer-word.test.ts index 009786a28c..4ea7679450 100644 --- a/codex-cli/tests/text-buffer-word.test.ts +++ b/codex-cli/tests/text-buffer-word.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, test, expect } from "vitest"; describe("TextBuffer – word‑wise navigation & deletion", () => { diff --git a/codex-cli/tests/text-buffer.test.ts b/codex-cli/tests/text-buffer.test.ts index ae78f29409..c3f33d0fa1 100644 --- a/codex-cli/tests/text-buffer.test.ts +++ b/codex-cli/tests/text-buffer.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; describe("TextBuffer – basic editing parity with Rust suite", () => { diff --git a/codex-cli/tsconfig.json b/codex-cli/tsconfig.json index 626fc5dbf4..d1dacc9149 100644 --- a/codex-cli/tsconfig.json +++ b/codex-cli/tsconfig.json @@ -11,9 +11,6 @@ ], "types": ["node"], "baseUrl": "./", - "paths": { - "@lib/*": ["./src/lib/*"] - }, "resolveJsonModule": false, // ESM doesn't yet support JSON modules. "jsx": "react", "declaration": true, From 2cafcf2ec061552d195a762123025d21f0f00097 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 14:15:00 -0700 Subject: [PATCH 03/84] Back out @lib indirection in tsconfig.json --- codex-cli/src/app.tsx | 2 +- codex-cli/src/{lib => }/approvals.ts | 2 +- codex-cli/src/cli.tsx | 4 +- .../src/components/chat/multiline-editor.tsx | 2 +- .../chat/terminal-chat-tool-call-item.tsx | 2 +- .../src/components/chat/terminal-chat.tsx | 4 +- codex-cli/src/{lib => }/format-command.ts | 0 codex-cli/src/{lib => }/parse-apply-patch.ts | 0 codex-cli/src/{lib => }/text-buffer.ts | 0 codex-cli/src/utils/agent/agent-loop.ts | 2 +- codex-cli/src/utils/agent/exec.ts | 2 +- .../src/utils/agent/handle-exec-command.ts | 6 +- codex-cli/src/utils/agent/review.ts | 2 +- codex-cli/src/utils/parsers.ts | 55 ++----------------- codex-cli/tests/agent-cancel-early.test.ts | 4 +- .../tests/agent-cancel-prev-response.test.ts | 4 +- codex-cli/tests/agent-cancel-race.test.ts | 4 +- codex-cli/tests/agent-cancel.test.ts | 4 +- .../tests/agent-function-call-id.test.ts | 4 +- .../tests/agent-generic-network-error.test.ts | 4 +- .../tests/agent-invalid-request-error.test.ts | 4 +- .../tests/agent-max-tokens-error.test.ts | 4 +- codex-cli/tests/agent-network-errors.test.ts | 4 +- codex-cli/tests/agent-project-doc.test.ts | 4 +- .../tests/agent-rate-limit-error.test.ts | 4 +- codex-cli/tests/agent-server-retry.test.ts | 4 +- codex-cli/tests/agent-terminate.test.ts | 4 +- codex-cli/tests/agent-thinking-time.test.ts | 4 +- codex-cli/tests/approvals.test.ts | 4 +- codex-cli/tests/external-editor.test.ts | 2 +- codex-cli/tests/format-command.test.ts | 2 +- .../tests/invalid-command-handling.test.ts | 4 +- ...ultiline-external-editor-shortcut.test.tsx | 2 +- .../tests/multiline-history-behavior.test.tsx | 6 +- codex-cli/tests/parse-apply-patch.test.ts | 2 +- .../tests/text-buffer-copy-paste.test.ts | 2 +- codex-cli/tests/text-buffer-crlf.test.ts | 2 +- codex-cli/tests/text-buffer-gaps.test.ts | 2 +- codex-cli/tests/text-buffer-word.test.ts | 2 +- codex-cli/tests/text-buffer.test.ts | 2 +- codex-cli/tsconfig.json | 3 - 41 files changed, 63 insertions(+), 111 deletions(-) rename codex-cli/src/{lib => }/approvals.ts (99%) rename codex-cli/src/{lib => }/format-command.ts (100%) rename codex-cli/src/{lib => }/parse-apply-patch.ts (100%) rename codex-cli/src/{lib => }/text-buffer.ts (100%) diff --git a/codex-cli/src/app.tsx b/codex-cli/src/app.tsx index dbb0cdedc8..c0b8c6f4e3 100644 --- a/codex-cli/src/app.tsx +++ b/codex-cli/src/app.tsx @@ -1,5 +1,5 @@ +import type { ApprovalPolicy } from "./approvals"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import TerminalChat from "./components/chat/terminal-chat"; diff --git a/codex-cli/src/lib/approvals.ts b/codex-cli/src/approvals.ts similarity index 99% rename from codex-cli/src/lib/approvals.ts rename to codex-cli/src/approvals.ts index 8985939a43..0cf3703b54 100644 --- a/codex-cli/src/lib/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -3,7 +3,7 @@ import type { ParseEntry, ControlOperator } from "shell-quote"; import { identify_files_added, identify_files_needed, -} from "../utils/agent/apply-patch"; +} from "./utils/agent/apply-patch"; import * as path from "path"; import { parse } from "shell-quote"; diff --git a/codex-cli/src/cli.tsx b/codex-cli/src/cli.tsx index 5e43dff1f3..0af421a431 100644 --- a/codex-cli/src/cli.tsx +++ b/codex-cli/src/cli.tsx @@ -1,9 +1,9 @@ #!/usr/bin/env node import type { AppRollout } from "./app"; +import type { ApprovalPolicy } from "./approvals"; import type { CommandConfirmation } from "./utils/agent/agent-loop"; import type { AppConfig } from "./utils/config"; -import type { ApprovalPolicy } from "@lib/approvals"; import type { ResponseItem } from "openai/resources/responses/responses"; import App from "./app"; @@ -124,7 +124,7 @@ const cli = meow( fullContext: { type: "boolean", aliases: ["f"], - description: `Run in full-context editing approach. The model is given the whole code + description: `Run in full-context editing approach. The model is given the whole code directory as context and performs changes in one go without acting.`, }, }, diff --git a/codex-cli/src/components/chat/multiline-editor.tsx b/codex-cli/src/components/chat/multiline-editor.tsx index c18555447e..c99961bbcd 100644 --- a/codex-cli/src/components/chat/multiline-editor.tsx +++ b/codex-cli/src/components/chat/multiline-editor.tsx @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { useTerminalSize } from "../../hooks/use-terminal-size"; -import TextBuffer from "../../lib/text-buffer.js"; +import TextBuffer from "../../text-buffer.js"; import chalk from "chalk"; import { Box, Text, useInput, useStdin } from "ink"; import { EventEmitter } from "node:events"; diff --git a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx index 1aeb7d7e98..5853460884 100644 --- a/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx +++ b/codex-cli/src/components/chat/terminal-chat-tool-call-item.tsx @@ -1,5 +1,5 @@ +import { parseApplyPatch } from "../../parse-apply-patch"; import { shortenPath } from "../../utils/short-path"; -import { parseApplyPatch } from "@lib/parse-apply-patch"; import chalk from "chalk"; import { Text } from "ink"; import React from "react"; diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index bbc9bb052a..35fecec564 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -1,6 +1,6 @@ +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { CommandConfirmation } from "../../utils/agent/agent-loop.js"; import type { AppConfig } from "../../utils/config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ColorName } from "chalk"; import type { ResponseItem } from "openai/resources/responses/responses.mjs"; import type { ReviewDecision } from "src/utils/agent/review.ts"; @@ -12,6 +12,7 @@ import { uniqueById, } from "./terminal-chat-utils.js"; import TerminalMessageHistory from "./terminal-message-history.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; @@ -25,7 +26,6 @@ import ApprovalModeOverlay from "../approval-mode-overlay.js"; import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; import { Box, Text } from "ink"; import React, { useEffect, useMemo, useState } from "react"; import { inspect } from "util"; diff --git a/codex-cli/src/lib/format-command.ts b/codex-cli/src/format-command.ts similarity index 100% rename from codex-cli/src/lib/format-command.ts rename to codex-cli/src/format-command.ts diff --git a/codex-cli/src/lib/parse-apply-patch.ts b/codex-cli/src/parse-apply-patch.ts similarity index 100% rename from codex-cli/src/lib/parse-apply-patch.ts rename to codex-cli/src/parse-apply-patch.ts diff --git a/codex-cli/src/lib/text-buffer.ts b/codex-cli/src/text-buffer.ts similarity index 100% rename from codex-cli/src/lib/text-buffer.ts rename to codex-cli/src/text-buffer.ts diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index e2b465733d..d88604c471 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -1,6 +1,6 @@ import type { ReviewDecision } from "./review.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { AppConfig } from "../config.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; import type { ResponseFunctionToolCall, ResponseInputItem, diff --git a/codex-cli/src/utils/agent/exec.ts b/codex-cli/src/utils/agent/exec.ts index aade68a860..a441f192f8 100644 --- a/codex-cli/src/utils/agent/exec.ts +++ b/codex-cli/src/utils/agent/exec.ts @@ -5,7 +5,7 @@ import { process_patch } from "./apply-patch.js"; import { SandboxType } from "./sandbox/interface.js"; import { execWithSeatbelt } from "./sandbox/macos-seatbelt.js"; import { exec as rawExec } from "./sandbox/raw-exec.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import fs from "fs"; import os from "os"; diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 5eaa11d660..41b7abbcd4 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -1,7 +1,7 @@ import type { CommandConfirmation } from "./agent-loop.js"; import type { AppConfig } from "../config.js"; import type { ExecInput } from "./sandbox/interface.js"; -import type { ApplyPatchCommand, ApprovalPolicy } from "@lib/approvals.js"; +import type { ApplyPatchCommand, ApprovalPolicy } from "../../approvals.js"; import type { ResponseInputItem } from "openai/resources/responses/responses.mjs"; import { exec, execApplyPatch } from "./exec.js"; @@ -9,8 +9,8 @@ import { isLoggingEnabled, log } from "./log.js"; import { ReviewDecision } from "./review.js"; import { FullAutoErrorMode } from "../auto-approval-mode.js"; import { SandboxType } from "./sandbox/interface.js"; -import { canAutoApprove } from "@lib/approvals.js"; -import { formatCommandForDisplay } from "@lib/format-command.js"; +import { canAutoApprove } from "../../approvals.js"; +import { formatCommandForDisplay } from "../../format-command.js"; import { access } from "fs/promises"; // --------------------------------------------------------------------------- diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index ed2af0ce17..a370388569 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,4 +1,4 @@ -import type { SafeCommandReason } from "@lib/approvals"; +import type { SafeCommandReason } from "../../approvals"; export type CommandReviewDetails = { cmd: Array; diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index cb477c1d7d..815e7b2f00 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -3,11 +3,13 @@ import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; -import type { SafeCommandReason } from "@lib/approvals.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; +import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; import process from "process"; +import { parse } from "shell-quote"; +import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is @@ -23,52 +25,6 @@ const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ ";", ]); -// Lazily resolve heavy dependencies at runtime to avoid test environments -// (which might not have the @lib alias configured) from failing at import -// time. If the modules cannot be loaded we fall back to permissive stub -// implementations so that basic functionality – like unit‑testing small UI -// helpers – continues to work without the full codex‑lib dependency tree. - -let isSafeCommand: (cmd: Array) => SafeCommandReason | null = () => - null; -let shellQuoteParse: - | ((cmd: string, env?: Record) => Array) - | undefined; -let formatCommandForDisplay: (cmd: Array) => string = (cmd) => - cmd.join(" "); - -async function loadLibs(): Promise { - try { - const approvals = await import("@lib/approvals.js"); - if (typeof approvals.isSafeCommand === "function") { - isSafeCommand = approvals.isSafeCommand; - } - } catch { - // ignore – keep stub - } - try { - const fmt = await import("@lib/format-command.js"); - if (typeof fmt.formatCommandForDisplay === "function") { - formatCommandForDisplay = fmt.formatCommandForDisplay; - } - } catch { - // ignore – keep stub - } - try { - const sq = await import("shell-quote"); - if (typeof sq.parse === "function") { - shellQuoteParse = sq.parse as typeof shellQuoteParse; - } - } catch { - // ignore – keep stub - } -} - -// Trigger the dynamic import in the background; callers that need the real -// implementation should await the returned promise (parsers currently does not -// require this for correctness during tests). -void loadLibs(); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -175,10 +131,9 @@ function computeAutoApproval(cmd: Array): SafeCommandReason | null { cmd.length === 3 && cmd[0] === "bash" && cmd[1] === "-lc" && - typeof cmd[2] === "string" && - shellQuoteParse + typeof cmd[2] === "string" ) { - const parsed = shellQuoteParse(cmd[2], process.env ?? {}); + const parsed = parse(cmd[2], process.env ?? {}); if (parsed.length === 0) { return null; } diff --git a/codex-cli/tests/agent-cancel-early.test.ts b/codex-cli/tests/agent-cancel-early.test.ts index b7b5fdcae7..b235a6d6bc 100644 --- a/codex-cli/tests/agent-cancel-early.test.ts +++ b/codex-cli/tests/agent-cancel-early.test.ts @@ -64,13 +64,13 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-prev-response.test.ts b/codex-cli/tests/agent-cancel-prev-response.test.ts index 4047f88452..fe73c338cc 100644 --- a/codex-cli/tests/agent-cancel-prev-response.test.ts +++ b/codex-cli/tests/agent-cancel-prev-response.test.ts @@ -71,13 +71,13 @@ vi.mock("openai", () => { }); // Stub helpers not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel-race.test.ts b/codex-cli/tests/agent-cancel-race.test.ts index c9c1845d8e..89e7cca744 100644 --- a/codex-cli/tests/agent-cancel-race.test.ts +++ b/codex-cli/tests/agent-cancel-race.test.ts @@ -67,11 +67,11 @@ vi.mock("openai", () => { }); // Stubs for external helpers referenced indirectly. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-cancel.test.ts b/codex-cli/tests/agent-cancel.test.ts index 93ff5ba317..69c17f7f7e 100644 --- a/codex-cli/tests/agent-cancel.test.ts +++ b/codex-cli/tests/agent-cancel.test.ts @@ -47,7 +47,7 @@ vi.mock("openai", () => { }); // Mock the approvals and formatCommand helpers referenced by handle‑exec‑command. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -57,7 +57,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-function-call-id.test.ts b/codex-cli/tests/agent-function-call-id.test.ts index 7d0a55d122..d50c08eea4 100644 --- a/codex-cli/tests/agent-function-call-id.test.ts +++ b/codex-cli/tests/agent-function-call-id.test.ts @@ -88,14 +88,14 @@ vi.mock("openai", () => { }); // Stub approvals & command formatting – not relevant for this test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-generic-network-error.test.ts b/codex-cli/tests/agent-generic-network-error.test.ts index 8c636fbda3..942adff668 100644 --- a/codex-cli/tests/agent-generic-network-error.test.ts +++ b/codex-cli/tests/agent-generic-network-error.test.ts @@ -23,14 +23,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – unrelated to network handling. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-invalid-request-error.test.ts b/codex-cli/tests/agent-invalid-request-error.test.ts index 631d451944..090d0b52d9 100644 --- a/codex-cli/tests/agent-invalid-request-error.test.ts +++ b/codex-cli/tests/agent-invalid-request-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-max-tokens-error.test.ts b/codex-cli/tests/agent-max-tokens-error.test.ts index 5e01229454..de4fd17026 100644 --- a/codex-cli/tests/agent-max-tokens-error.test.ts +++ b/codex-cli/tests/agent-max-tokens-error.test.ts @@ -22,14 +22,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-network-errors.test.ts b/codex-cli/tests/agent-network-errors.test.ts index e01b08e918..f98ea5bf19 100644 --- a/codex-cli/tests/agent-network-errors.test.ts +++ b/codex-cli/tests/agent-network-errors.test.ts @@ -42,14 +42,14 @@ vi.mock("openai", () => { }); // Stub approvals / formatting helpers – not relevant here. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-project-doc.test.ts b/codex-cli/tests/agent-project-doc.test.ts index 97ad9c8c34..d3050f3953 100644 --- a/codex-cli/tests/agent-project-doc.test.ts +++ b/codex-cli/tests/agent-project-doc.test.ts @@ -51,7 +51,7 @@ vi.mock("openai", () => { // The AgentLoop pulls these helpers in order to decide whether a command can // be auto‑approved. None of that matters for this test, so we stub the module // with minimal no‑op implementations. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -61,7 +61,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-rate-limit-error.test.ts b/codex-cli/tests/agent-rate-limit-error.test.ts index 18779450aa..9782744679 100644 --- a/codex-cli/tests/agent-rate-limit-error.test.ts +++ b/codex-cli/tests/agent-rate-limit-error.test.ts @@ -34,14 +34,14 @@ vi.mock("openai", () => { // Stub helpers that the agent indirectly imports so it does not attempt any // file‑system access or real approvals logic during the test. -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-server-retry.test.ts b/codex-cli/tests/agent-server-retry.test.ts index 9ec4eb5aac..09278f2ceb 100644 --- a/codex-cli/tests/agent-server-retry.test.ts +++ b/codex-cli/tests/agent-server-retry.test.ts @@ -32,14 +32,14 @@ vi.mock("openai", () => { }; }); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, alwaysApprovedCommands: new Set(), canAutoApprove: () => ({ type: "auto-approve", runInSandbox: false } as any), isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/agent-terminate.test.ts b/codex-cli/tests/agent-terminate.test.ts index 7a60456518..bce77437af 100644 --- a/codex-cli/tests/agent-terminate.test.ts +++ b/codex-cli/tests/agent-terminate.test.ts @@ -49,7 +49,7 @@ vi.mock("openai", () => { // --- Helpers referenced by handle‑exec‑command ----------------------------- -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, alwaysApprovedCommands: new Set(), @@ -59,7 +59,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/agent-thinking-time.test.ts b/codex-cli/tests/agent-thinking-time.test.ts index c94d8a5e5e..7132070084 100644 --- a/codex-cli/tests/agent-thinking-time.test.ts +++ b/codex-cli/tests/agent-thinking-time.test.ts @@ -74,12 +74,12 @@ vi.mock("openai", () => { }); // Stub helpers referenced indirectly so we do not pull in real FS/network -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ __esModule: true, isSafeCommand: () => null, })); -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ __esModule: true, formatCommandForDisplay: (c: Array) => c.join(" "), })); diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index e06db94092..7cb0bd3d3e 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -1,6 +1,6 @@ -import type { SafetyAssessment } from "../src/lib/approvals"; +import type { SafetyAssessment } from "../src/approvals"; -import { canAutoApprove } from "../src/lib/approvals"; +import { canAutoApprove } from "../src/approvals"; import { describe, test, expect } from "vitest"; describe("canAutoApprove()", () => { diff --git a/codex-cli/tests/external-editor.test.ts b/codex-cli/tests/external-editor.test.ts index d530be5ecc..77041c2870 100644 --- a/codex-cli/tests/external-editor.test.ts +++ b/codex-cli/tests/external-editor.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect, vi } from "vitest"; /* ------------------------------------------------------------------------- diff --git a/codex-cli/tests/format-command.test.ts b/codex-cli/tests/format-command.test.ts index 3981d9b106..7de417308a 100644 --- a/codex-cli/tests/format-command.test.ts +++ b/codex-cli/tests/format-command.test.ts @@ -1,4 +1,4 @@ -import { formatCommandForDisplay } from "../src/lib/format-command"; +import { formatCommandForDisplay } from "../src/format-command"; import { describe, test, expect } from "vitest"; describe("formatCommandForDisplay()", () => { diff --git a/codex-cli/tests/invalid-command-handling.test.ts b/codex-cli/tests/invalid-command-handling.test.ts index 6619b4f240..a3f87a7251 100644 --- a/codex-cli/tests/invalid-command-handling.test.ts +++ b/codex-cli/tests/invalid-command-handling.test.ts @@ -22,7 +22,7 @@ describe("rawExec – invalid command handling", () => { // --------------------------------------------------------------------------- // Mock approvals and logging helpers so the test focuses on execution flow. -vi.mock("@lib/approvals.js", () => { +vi.mock("../src/approvals.js", () => { return { __esModule: true, canAutoApprove: () => @@ -31,7 +31,7 @@ vi.mock("@lib/approvals.js", () => { }; }); -vi.mock("@lib/format-command.js", () => { +vi.mock("../src/format-command.js", () => { return { __esModule: true, formatCommandForDisplay: (cmd: Array) => cmd.join(" "), diff --git a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx index 158a5e655d..9b2e2f25e5 100644 --- a/codex-cli/tests/multiline-external-editor-shortcut.test.tsx +++ b/codex-cli/tests/multiline-external-editor-shortcut.test.tsx @@ -1,6 +1,6 @@ import { renderTui } from "./ui-test-helpers.js"; import MultilineTextEditor from "../src/components/chat/multiline-editor.js"; -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import * as React from "react"; import { describe, it, expect, vi } from "vitest"; diff --git a/codex-cli/tests/multiline-history-behavior.test.tsx b/codex-cli/tests/multiline-history-behavior.test.tsx index 5c906837ee..cada52ddab 100644 --- a/codex-cli/tests/multiline-history-behavior.test.tsx +++ b/codex-cli/tests/multiline-history-behavior.test.tsx @@ -34,12 +34,12 @@ vi.mock("../src/utils/input-utils.js", () => ({ })), })); -// Mock the optional @lib/* dependencies so the dynamic import in parsers.ts +// Mock the optional ../src/* dependencies so the dynamic import in parsers.ts // does not fail during the test environment where the alias isn't configured. -vi.mock("@lib/format-command.js", () => ({ +vi.mock("../src/format-command.js", () => ({ formatCommandForDisplay: (cmd: Array) => cmd.join(" "), })); -vi.mock("@lib/approvals.js", () => ({ +vi.mock("../src/approvals.js", () => ({ isSafeCommand: (_cmd: Array) => null, })); diff --git a/codex-cli/tests/parse-apply-patch.test.ts b/codex-cli/tests/parse-apply-patch.test.ts index 0195542e56..53aa119093 100644 --- a/codex-cli/tests/parse-apply-patch.test.ts +++ b/codex-cli/tests/parse-apply-patch.test.ts @@ -1,4 +1,4 @@ -import { parseApplyPatch } from "../src/lib/parse-apply-patch"; +import { parseApplyPatch } from "../src/parse-apply-patch"; import { expect, test, describe } from "vitest"; // Helper function to unwrap a non‑null result in tests that expect success. diff --git a/codex-cli/tests/text-buffer-copy-paste.test.ts b/codex-cli/tests/text-buffer-copy-paste.test.ts index 311b2b9aa6..cc1fd119e5 100644 --- a/codex-cli/tests/text-buffer-copy-paste.test.ts +++ b/codex-cli/tests/text-buffer-copy-paste.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; // These tests ensure that the TextBuffer copy‑&‑paste logic keeps parity with diff --git a/codex-cli/tests/text-buffer-crlf.test.ts b/codex-cli/tests/text-buffer-crlf.test.ts index 4b33b498b5..736c22a27d 100644 --- a/codex-cli/tests/text-buffer-crlf.test.ts +++ b/codex-cli/tests/text-buffer-crlf.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, it, expect } from "vitest"; describe("TextBuffer – newline normalisation", () => { diff --git a/codex-cli/tests/text-buffer-gaps.test.ts b/codex-cli/tests/text-buffer-gaps.test.ts index 986ad37eed..046d468e8a 100644 --- a/codex-cli/tests/text-buffer-gaps.test.ts +++ b/codex-cli/tests/text-buffer-gaps.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; // The purpose of this test‑suite is NOT to make the implementation green today diff --git a/codex-cli/tests/text-buffer-word.test.ts b/codex-cli/tests/text-buffer-word.test.ts index 009786a28c..4ea7679450 100644 --- a/codex-cli/tests/text-buffer-word.test.ts +++ b/codex-cli/tests/text-buffer-word.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer.js"; +import TextBuffer from "../src/text-buffer.js"; import { describe, test, expect } from "vitest"; describe("TextBuffer – word‑wise navigation & deletion", () => { diff --git a/codex-cli/tests/text-buffer.test.ts b/codex-cli/tests/text-buffer.test.ts index ae78f29409..c3f33d0fa1 100644 --- a/codex-cli/tests/text-buffer.test.ts +++ b/codex-cli/tests/text-buffer.test.ts @@ -1,4 +1,4 @@ -import TextBuffer from "../src/lib/text-buffer"; +import TextBuffer from "../src/text-buffer"; import { describe, it, expect } from "vitest"; describe("TextBuffer – basic editing parity with Rust suite", () => { diff --git a/codex-cli/tsconfig.json b/codex-cli/tsconfig.json index 626fc5dbf4..d1dacc9149 100644 --- a/codex-cli/tsconfig.json +++ b/codex-cli/tsconfig.json @@ -11,9 +11,6 @@ ], "types": ["node"], "baseUrl": "./", - "paths": { - "@lib/*": ["./src/lib/*"] - }, "resolveJsonModule": false, // ESM doesn't yet support JSON modules. "jsx": "react", "declaration": true, From 54ba90c4b80cc820ca3267e93a2dd8dbf681c52f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 15:09:12 -0700 Subject: [PATCH 04/84] Removes computeAutoApproval() and tightens up canAutoApprove() as the source of truth --- codex-cli/src/approvals.ts | 127 ++++++++++-------- .../components/chat/use-message-grouping.ts | 72 ---------- codex-cli/src/utils/agent/review.ts | 8 -- codex-cli/src/utils/parsers.ts | 111 ++------------- 4 files changed, 80 insertions(+), 238 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index 0cf3703b54..8a670b01ca 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -75,19 +75,72 @@ export function canAutoApprove( writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { - try { - if (command[0] === "apply_patch") { - return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) - : { - type: "reject", - reason: "Invalid apply_patch command", - }; + if (command[0] === "apply_patch") { + return command.length === 2 && typeof command[1] === "string" + ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + : { + type: "reject", + reason: "Invalid apply_patch command", + }; + } + + const isSafe = isSafeCommand(command); + if (isSafe != null) { + const { reason, group } = isSafe; + return { + type: "auto-approve", + reason, + group, + runInSandbox: false, + }; + } + + if ( + command[0] === "bash" && + command[1] === "-lc" && + typeof command[2] === "string" && + command.length === 3 + ) { + const applyPatchArg = tryParseApplyPatch(command[2]); + if (applyPatchArg != null) { + return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); } - const isSafe = isSafeCommand(command); - if (isSafe != null) { - const { reason, group } = isSafe; + let bashCmd; + try { + bashCmd = parse(command[2], env); + } catch (e) { + // In practice, there seem to be syntactically valid shell commands that + // shell-quote cannot parse, so we should not reject, but ask the user. + switch (policy) { + case "full-auto": + // In full-auto, we still run the command automatically, but must + // restrict it to the sandbox. + return { + type: "auto-approve", + reason: "Full auto mode", + group: "Running commands", + runInSandbox: true, + }; + case "suggest": + case "auto-edit": + // In all other modes, since we cannot reason about the command, we + // should ask the user. + return { + type: "ask-user", + }; + } + } + + // bashCmd could be a mix of strings and operators, e.g.: + // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] + // We try to ensure that *every* command segment is deemed safe and that + // all operators belong to an allow‑list. If so, the entire expression is + // considered auto‑approvable. + + const shellSafe = isEntireShellExpressionSafe(bashCmd); + if (shellSafe != null) { + const { reason, group } = shellSafe; return { type: "auto-approve", reason, @@ -95,58 +148,16 @@ export function canAutoApprove( runInSandbox: false, }; } + } - if ( - command[0] === "bash" && - command[1] === "-lc" && - typeof command[2] === "string" && - command.length === 3 - ) { - const applyPatchArg = tryParseApplyPatch(command[2]); - if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); - } - - const bashCmd = parse(command[2], env); - - // bashCmd could be a mix of strings and operators, e.g.: - // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] - // We try to ensure that *every* command segment is deemed safe and that - // all operators belong to an allow‑list. If so, the entire expression is - // considered auto‑approvable. - - const shellSafe = isEntireShellExpressionSafe(bashCmd); - if (shellSafe != null) { - const { reason, group } = shellSafe; - return { - type: "auto-approve", - reason, - group, - runInSandbox: false, - }; - } - } - - return policy === "full-auto" - ? { - type: "auto-approve", - reason: "Full auto mode", - group: "Running commands", - runInSandbox: true, - } - : { type: "ask-user" }; - } catch (err) { - if (policy === "full-auto") { - return { + return policy === "full-auto" + ? { type: "auto-approve", reason: "Full auto mode", group: "Running commands", runInSandbox: true, - }; - } else { - return { type: "ask-user" }; - } - } + } + : { type: "ask-user" }; } function canAutoApproveApplyPatch( diff --git a/codex-cli/src/components/chat/use-message-grouping.ts b/codex-cli/src/components/chat/use-message-grouping.ts index 75e51ac682..1e526821d0 100644 --- a/codex-cli/src/components/chat/use-message-grouping.ts +++ b/codex-cli/src/components/chat/use-message-grouping.ts @@ -1,8 +1,5 @@ import type { ResponseItem } from "openai/resources/responses/responses.mjs"; -import { parseToolCall } from "../../utils/parsers.js"; -import { useMemo } from "react"; - /** * Represents a grouped sequence of response items (e.g., function call batches). */ @@ -10,72 +7,3 @@ export type GroupedResponseItem = { label: string; items: Array; }; - -/** - * Custom hook to group recent response items for display batching. - * Returns counts of auto-approved tool call groups, the latest batch, - * and the count of user messages in the visible window. - */ -export function useMessageGrouping(visibleItems: Array): { - groupCounts: Record; - batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }>; - userMsgCount: number; -} { - return useMemo(() => { - // The grouping logic only depends on the subset of messages that are - // currently rendered (visibleItems). Using that as the sole dependency - // keeps recomputations to a minimum and avoids unnecessary work when the - // full list of `items` changes outside of the visible window. - let userMsgCount = 0; - const groupCounts: Record = {}; - visibleItems.forEach((m) => { - if (m.type === "function_call") { - const toolCall = parseToolCall(m); - if (toolCall?.autoApproval) { - const group = toolCall.autoApproval.group; - groupCounts[group] = (groupCounts[group] || 0) + 1; - } - } - if (m.type === "message" && m.role === "user") { - userMsgCount++; - } - }); - const lastFew = visibleItems.slice(-3); - const batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }> = - []; - if (lastFew[0]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[0]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew, - }, - }); - if (lastFew[2]?.type === "message") { - batch.push({ item: lastFew[2] }); - } - } else if (lastFew[1]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[1]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew.slice(1), - }, - }); - } else if (lastFew[2]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[2]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: [lastFew[2]], - }, - }); - } else { - lastFew.forEach((item) => batch.push({ item })); - } - return { groupCounts, batch, userMsgCount }; - // `items` is stable across renders while `visibleItems` changes based on - // the scroll window. Including only `visibleItems` avoids unnecessary - // recomputations while still producing correct results. - }, [visibleItems]); -} diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index a370388569..9a5e66a4ff 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,11 +1,3 @@ -import type { SafeCommandReason } from "../../approvals"; - -export type CommandReviewDetails = { - cmd: Array; - cmdReadableText: string; - autoApproval: SafeCommandReason | null; -}; - export enum ReviewDecision { YES = "yes", NO_CONTINUE = "no-continue", diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index 815e7b2f00..e46db3b34d 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -5,26 +5,13 @@ import type { } from "./agent/sandbox/interface.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; -import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; -import process from "process"; -import { parse } from "shell-quote"; import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is // required. -// Allowed shell operators that we consider "safe" as they do not introduce -// side‑effects on their own (unlike redirections). Parentheses and braces for -// grouping are excluded for simplicity. -const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ - "&&", - "||", - "|", - ";", -]); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -46,6 +33,17 @@ export function parseToolCallOutput(toolCallOutput: string): { } } +export type CommandReviewDetails = { + cmd: Array; + cmdReadableText: string; +}; + +/** + * Tries to parse a tool call and, if successful, returns an object that has + * both: + * - an array of strings to use with `ExecInput` and `canAutoApprove()` + * - a human-readable string to display to the user + */ export function parseToolCall( toolCall: ResponseFunctionToolCall, ): CommandReviewDetails | undefined { @@ -57,12 +55,9 @@ export function parseToolCall( const { cmd } = toolCallArgs; const cmdReadableText = formatCommandForDisplay(cmd); - const autoApproval = computeAutoApproval(cmd); - return { cmd, cmdReadableText, - autoApproval, }; } @@ -109,87 +104,3 @@ function toStringArray(obj: unknown): Array | undefined { return undefined; } } - -// ---------------- safe‑command helpers ---------------- - -/** - * Attempts to determine whether `cmd` is composed exclusively of safe - * sub‑commands combined using only operators from the SAFE_SHELL_OPERATORS - * allow‑list. Returns the `SafeCommandReason` (taken from the first sub‑command) - * if the whole expression is safe; otherwise returns `null`. - */ -function computeAutoApproval(cmd: Array): SafeCommandReason | null { - // Fast path: a simple command with no shell processing. - const direct = isSafeCommand(cmd); - if (direct != null) { - return direct; - } - - // For expressions like ["bash", "-lc", "ls && pwd"] break down the inner - // string and verify each segment. - if ( - cmd.length === 3 && - cmd[0] === "bash" && - cmd[1] === "-lc" && - typeof cmd[2] === "string" - ) { - const parsed = parse(cmd[2], process.env ?? {}); - if (parsed.length === 0) { - return null; - } - - let current: Array = []; - let first: SafeCommandReason | null = null; - - const flush = (): boolean => { - if (current.length === 0) { - return true; - } - const safe = isSafeCommand(current); - if (safe == null) { - return false; - } - if (!first) { - first = safe; - } - current = []; - return true; - }; - - for (const part of parsed) { - if (typeof part === "string") { - // Simple word/argument token. - if (part === "(" || part === ")" || part === "{" || part === "}") { - // We treat explicit grouping tokens as unsafe because their - // semantics depend on the shell evaluation environment. - return null; - } - current.push(part); - } else if (part && typeof part === "object") { - const opToken = part as { op?: string }; - if (typeof opToken.op === "string") { - if (!flush()) { - return null; - } - if (!SAFE_SHELL_OPERATORS.has(opToken.op)) { - return null; - } - } else { - // Unknown object token kind (e.g. redirection) – treat as unsafe. - return null; - } - } else { - // Token types such as numbers / booleans are unexpected – treat as unsafe. - return null; - } - } - - if (!flush()) { - return null; - } - - return first; - } - - return null; -} From c56f84c5f59ddd36cb92a46bf184773f1bc5c318 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 15:09:12 -0700 Subject: [PATCH 05/84] Removes computeAutoApproval() and tightens up canAutoApprove() as the source of truth --- codex-cli/src/approvals.ts | 127 ++++++++++-------- .../components/chat/use-message-grouping.ts | 72 ---------- codex-cli/src/utils/agent/review.ts | 8 -- codex-cli/src/utils/parsers.ts | 112 ++------------- 4 files changed, 80 insertions(+), 239 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index 0cf3703b54..8a670b01ca 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -75,19 +75,72 @@ export function canAutoApprove( writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { - try { - if (command[0] === "apply_patch") { - return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) - : { - type: "reject", - reason: "Invalid apply_patch command", - }; + if (command[0] === "apply_patch") { + return command.length === 2 && typeof command[1] === "string" + ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + : { + type: "reject", + reason: "Invalid apply_patch command", + }; + } + + const isSafe = isSafeCommand(command); + if (isSafe != null) { + const { reason, group } = isSafe; + return { + type: "auto-approve", + reason, + group, + runInSandbox: false, + }; + } + + if ( + command[0] === "bash" && + command[1] === "-lc" && + typeof command[2] === "string" && + command.length === 3 + ) { + const applyPatchArg = tryParseApplyPatch(command[2]); + if (applyPatchArg != null) { + return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); } - const isSafe = isSafeCommand(command); - if (isSafe != null) { - const { reason, group } = isSafe; + let bashCmd; + try { + bashCmd = parse(command[2], env); + } catch (e) { + // In practice, there seem to be syntactically valid shell commands that + // shell-quote cannot parse, so we should not reject, but ask the user. + switch (policy) { + case "full-auto": + // In full-auto, we still run the command automatically, but must + // restrict it to the sandbox. + return { + type: "auto-approve", + reason: "Full auto mode", + group: "Running commands", + runInSandbox: true, + }; + case "suggest": + case "auto-edit": + // In all other modes, since we cannot reason about the command, we + // should ask the user. + return { + type: "ask-user", + }; + } + } + + // bashCmd could be a mix of strings and operators, e.g.: + // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] + // We try to ensure that *every* command segment is deemed safe and that + // all operators belong to an allow‑list. If so, the entire expression is + // considered auto‑approvable. + + const shellSafe = isEntireShellExpressionSafe(bashCmd); + if (shellSafe != null) { + const { reason, group } = shellSafe; return { type: "auto-approve", reason, @@ -95,58 +148,16 @@ export function canAutoApprove( runInSandbox: false, }; } + } - if ( - command[0] === "bash" && - command[1] === "-lc" && - typeof command[2] === "string" && - command.length === 3 - ) { - const applyPatchArg = tryParseApplyPatch(command[2]); - if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); - } - - const bashCmd = parse(command[2], env); - - // bashCmd could be a mix of strings and operators, e.g.: - // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] - // We try to ensure that *every* command segment is deemed safe and that - // all operators belong to an allow‑list. If so, the entire expression is - // considered auto‑approvable. - - const shellSafe = isEntireShellExpressionSafe(bashCmd); - if (shellSafe != null) { - const { reason, group } = shellSafe; - return { - type: "auto-approve", - reason, - group, - runInSandbox: false, - }; - } - } - - return policy === "full-auto" - ? { - type: "auto-approve", - reason: "Full auto mode", - group: "Running commands", - runInSandbox: true, - } - : { type: "ask-user" }; - } catch (err) { - if (policy === "full-auto") { - return { + return policy === "full-auto" + ? { type: "auto-approve", reason: "Full auto mode", group: "Running commands", runInSandbox: true, - }; - } else { - return { type: "ask-user" }; - } - } + } + : { type: "ask-user" }; } function canAutoApproveApplyPatch( diff --git a/codex-cli/src/components/chat/use-message-grouping.ts b/codex-cli/src/components/chat/use-message-grouping.ts index 75e51ac682..1e526821d0 100644 --- a/codex-cli/src/components/chat/use-message-grouping.ts +++ b/codex-cli/src/components/chat/use-message-grouping.ts @@ -1,8 +1,5 @@ import type { ResponseItem } from "openai/resources/responses/responses.mjs"; -import { parseToolCall } from "../../utils/parsers.js"; -import { useMemo } from "react"; - /** * Represents a grouped sequence of response items (e.g., function call batches). */ @@ -10,72 +7,3 @@ export type GroupedResponseItem = { label: string; items: Array; }; - -/** - * Custom hook to group recent response items for display batching. - * Returns counts of auto-approved tool call groups, the latest batch, - * and the count of user messages in the visible window. - */ -export function useMessageGrouping(visibleItems: Array): { - groupCounts: Record; - batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }>; - userMsgCount: number; -} { - return useMemo(() => { - // The grouping logic only depends on the subset of messages that are - // currently rendered (visibleItems). Using that as the sole dependency - // keeps recomputations to a minimum and avoids unnecessary work when the - // full list of `items` changes outside of the visible window. - let userMsgCount = 0; - const groupCounts: Record = {}; - visibleItems.forEach((m) => { - if (m.type === "function_call") { - const toolCall = parseToolCall(m); - if (toolCall?.autoApproval) { - const group = toolCall.autoApproval.group; - groupCounts[group] = (groupCounts[group] || 0) + 1; - } - } - if (m.type === "message" && m.role === "user") { - userMsgCount++; - } - }); - const lastFew = visibleItems.slice(-3); - const batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }> = - []; - if (lastFew[0]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[0]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew, - }, - }); - if (lastFew[2]?.type === "message") { - batch.push({ item: lastFew[2] }); - } - } else if (lastFew[1]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[1]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew.slice(1), - }, - }); - } else if (lastFew[2]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[2]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: [lastFew[2]], - }, - }); - } else { - lastFew.forEach((item) => batch.push({ item })); - } - return { groupCounts, batch, userMsgCount }; - // `items` is stable across renders while `visibleItems` changes based on - // the scroll window. Including only `visibleItems` avoids unnecessary - // recomputations while still producing correct results. - }, [visibleItems]); -} diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index a370388569..9a5e66a4ff 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,11 +1,3 @@ -import type { SafeCommandReason } from "../../approvals"; - -export type CommandReviewDetails = { - cmd: Array; - cmdReadableText: string; - autoApproval: SafeCommandReason | null; -}; - export enum ReviewDecision { YES = "yes", NO_CONTINUE = "no-continue", diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index 815e7b2f00..4461379c02 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -1,30 +1,16 @@ -import type { CommandReviewDetails } from "./agent/review.js"; import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; -import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; -import process from "process"; -import { parse } from "shell-quote"; import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is // required. -// Allowed shell operators that we consider "safe" as they do not introduce -// side‑effects on their own (unlike redirections). Parentheses and braces for -// grouping are excluded for simplicity. -const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ - "&&", - "||", - "|", - ";", -]); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -46,6 +32,17 @@ export function parseToolCallOutput(toolCallOutput: string): { } } +export type CommandReviewDetails = { + cmd: Array; + cmdReadableText: string; +}; + +/** + * Tries to parse a tool call and, if successful, returns an object that has + * both: + * - an array of strings to use with `ExecInput` and `canAutoApprove()` + * - a human-readable string to display to the user + */ export function parseToolCall( toolCall: ResponseFunctionToolCall, ): CommandReviewDetails | undefined { @@ -57,12 +54,9 @@ export function parseToolCall( const { cmd } = toolCallArgs; const cmdReadableText = formatCommandForDisplay(cmd); - const autoApproval = computeAutoApproval(cmd); - return { cmd, cmdReadableText, - autoApproval, }; } @@ -109,87 +103,3 @@ function toStringArray(obj: unknown): Array | undefined { return undefined; } } - -// ---------------- safe‑command helpers ---------------- - -/** - * Attempts to determine whether `cmd` is composed exclusively of safe - * sub‑commands combined using only operators from the SAFE_SHELL_OPERATORS - * allow‑list. Returns the `SafeCommandReason` (taken from the first sub‑command) - * if the whole expression is safe; otherwise returns `null`. - */ -function computeAutoApproval(cmd: Array): SafeCommandReason | null { - // Fast path: a simple command with no shell processing. - const direct = isSafeCommand(cmd); - if (direct != null) { - return direct; - } - - // For expressions like ["bash", "-lc", "ls && pwd"] break down the inner - // string and verify each segment. - if ( - cmd.length === 3 && - cmd[0] === "bash" && - cmd[1] === "-lc" && - typeof cmd[2] === "string" - ) { - const parsed = parse(cmd[2], process.env ?? {}); - if (parsed.length === 0) { - return null; - } - - let current: Array = []; - let first: SafeCommandReason | null = null; - - const flush = (): boolean => { - if (current.length === 0) { - return true; - } - const safe = isSafeCommand(current); - if (safe == null) { - return false; - } - if (!first) { - first = safe; - } - current = []; - return true; - }; - - for (const part of parsed) { - if (typeof part === "string") { - // Simple word/argument token. - if (part === "(" || part === ")" || part === "{" || part === "}") { - // We treat explicit grouping tokens as unsafe because their - // semantics depend on the shell evaluation environment. - return null; - } - current.push(part); - } else if (part && typeof part === "object") { - const opToken = part as { op?: string }; - if (typeof opToken.op === "string") { - if (!flush()) { - return null; - } - if (!SAFE_SHELL_OPERATORS.has(opToken.op)) { - return null; - } - } else { - // Unknown object token kind (e.g. redirection) – treat as unsafe. - return null; - } - } else { - // Token types such as numbers / booleans are unexpected – treat as unsafe. - return null; - } - } - - if (!flush()) { - return null; - } - - return first; - } - - return null; -} From db9c62b85d40fd1fc44cadc0b3337da2c3e12c17 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 15:14:02 -0700 Subject: [PATCH 06/84] Removes computeAutoApproval() and tightens up canAutoApprove() as the source of truth --- codex-cli/src/approvals.ts | 127 ++++++++++-------- .../components/chat/use-message-grouping.ts | 72 ---------- codex-cli/src/utils/agent/review.ts | 8 -- codex-cli/src/utils/parsers.ts | 112 ++------------- 4 files changed, 80 insertions(+), 239 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index 0cf3703b54..8a670b01ca 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -75,19 +75,72 @@ export function canAutoApprove( writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { - try { - if (command[0] === "apply_patch") { - return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) - : { - type: "reject", - reason: "Invalid apply_patch command", - }; + if (command[0] === "apply_patch") { + return command.length === 2 && typeof command[1] === "string" + ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + : { + type: "reject", + reason: "Invalid apply_patch command", + }; + } + + const isSafe = isSafeCommand(command); + if (isSafe != null) { + const { reason, group } = isSafe; + return { + type: "auto-approve", + reason, + group, + runInSandbox: false, + }; + } + + if ( + command[0] === "bash" && + command[1] === "-lc" && + typeof command[2] === "string" && + command.length === 3 + ) { + const applyPatchArg = tryParseApplyPatch(command[2]); + if (applyPatchArg != null) { + return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); } - const isSafe = isSafeCommand(command); - if (isSafe != null) { - const { reason, group } = isSafe; + let bashCmd; + try { + bashCmd = parse(command[2], env); + } catch (e) { + // In practice, there seem to be syntactically valid shell commands that + // shell-quote cannot parse, so we should not reject, but ask the user. + switch (policy) { + case "full-auto": + // In full-auto, we still run the command automatically, but must + // restrict it to the sandbox. + return { + type: "auto-approve", + reason: "Full auto mode", + group: "Running commands", + runInSandbox: true, + }; + case "suggest": + case "auto-edit": + // In all other modes, since we cannot reason about the command, we + // should ask the user. + return { + type: "ask-user", + }; + } + } + + // bashCmd could be a mix of strings and operators, e.g.: + // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] + // We try to ensure that *every* command segment is deemed safe and that + // all operators belong to an allow‑list. If so, the entire expression is + // considered auto‑approvable. + + const shellSafe = isEntireShellExpressionSafe(bashCmd); + if (shellSafe != null) { + const { reason, group } = shellSafe; return { type: "auto-approve", reason, @@ -95,58 +148,16 @@ export function canAutoApprove( runInSandbox: false, }; } + } - if ( - command[0] === "bash" && - command[1] === "-lc" && - typeof command[2] === "string" && - command.length === 3 - ) { - const applyPatchArg = tryParseApplyPatch(command[2]); - if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); - } - - const bashCmd = parse(command[2], env); - - // bashCmd could be a mix of strings and operators, e.g.: - // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] - // We try to ensure that *every* command segment is deemed safe and that - // all operators belong to an allow‑list. If so, the entire expression is - // considered auto‑approvable. - - const shellSafe = isEntireShellExpressionSafe(bashCmd); - if (shellSafe != null) { - const { reason, group } = shellSafe; - return { - type: "auto-approve", - reason, - group, - runInSandbox: false, - }; - } - } - - return policy === "full-auto" - ? { - type: "auto-approve", - reason: "Full auto mode", - group: "Running commands", - runInSandbox: true, - } - : { type: "ask-user" }; - } catch (err) { - if (policy === "full-auto") { - return { + return policy === "full-auto" + ? { type: "auto-approve", reason: "Full auto mode", group: "Running commands", runInSandbox: true, - }; - } else { - return { type: "ask-user" }; - } - } + } + : { type: "ask-user" }; } function canAutoApproveApplyPatch( diff --git a/codex-cli/src/components/chat/use-message-grouping.ts b/codex-cli/src/components/chat/use-message-grouping.ts index 75e51ac682..1e526821d0 100644 --- a/codex-cli/src/components/chat/use-message-grouping.ts +++ b/codex-cli/src/components/chat/use-message-grouping.ts @@ -1,8 +1,5 @@ import type { ResponseItem } from "openai/resources/responses/responses.mjs"; -import { parseToolCall } from "../../utils/parsers.js"; -import { useMemo } from "react"; - /** * Represents a grouped sequence of response items (e.g., function call batches). */ @@ -10,72 +7,3 @@ export type GroupedResponseItem = { label: string; items: Array; }; - -/** - * Custom hook to group recent response items for display batching. - * Returns counts of auto-approved tool call groups, the latest batch, - * and the count of user messages in the visible window. - */ -export function useMessageGrouping(visibleItems: Array): { - groupCounts: Record; - batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }>; - userMsgCount: number; -} { - return useMemo(() => { - // The grouping logic only depends on the subset of messages that are - // currently rendered (visibleItems). Using that as the sole dependency - // keeps recomputations to a minimum and avoids unnecessary work when the - // full list of `items` changes outside of the visible window. - let userMsgCount = 0; - const groupCounts: Record = {}; - visibleItems.forEach((m) => { - if (m.type === "function_call") { - const toolCall = parseToolCall(m); - if (toolCall?.autoApproval) { - const group = toolCall.autoApproval.group; - groupCounts[group] = (groupCounts[group] || 0) + 1; - } - } - if (m.type === "message" && m.role === "user") { - userMsgCount++; - } - }); - const lastFew = visibleItems.slice(-3); - const batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }> = - []; - if (lastFew[0]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[0]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew, - }, - }); - if (lastFew[2]?.type === "message") { - batch.push({ item: lastFew[2] }); - } - } else if (lastFew[1]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[1]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew.slice(1), - }, - }); - } else if (lastFew[2]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[2]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: [lastFew[2]], - }, - }); - } else { - lastFew.forEach((item) => batch.push({ item })); - } - return { groupCounts, batch, userMsgCount }; - // `items` is stable across renders while `visibleItems` changes based on - // the scroll window. Including only `visibleItems` avoids unnecessary - // recomputations while still producing correct results. - }, [visibleItems]); -} diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index a370388569..9a5e66a4ff 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,11 +1,3 @@ -import type { SafeCommandReason } from "../../approvals"; - -export type CommandReviewDetails = { - cmd: Array; - cmdReadableText: string; - autoApproval: SafeCommandReason | null; -}; - export enum ReviewDecision { YES = "yes", NO_CONTINUE = "no-continue", diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index 815e7b2f00..4461379c02 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -1,30 +1,16 @@ -import type { CommandReviewDetails } from "./agent/review.js"; import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; -import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; -import process from "process"; -import { parse } from "shell-quote"; import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is // required. -// Allowed shell operators that we consider "safe" as they do not introduce -// side‑effects on their own (unlike redirections). Parentheses and braces for -// grouping are excluded for simplicity. -const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ - "&&", - "||", - "|", - ";", -]); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -46,6 +32,17 @@ export function parseToolCallOutput(toolCallOutput: string): { } } +export type CommandReviewDetails = { + cmd: Array; + cmdReadableText: string; +}; + +/** + * Tries to parse a tool call and, if successful, returns an object that has + * both: + * - an array of strings to use with `ExecInput` and `canAutoApprove()` + * - a human-readable string to display to the user + */ export function parseToolCall( toolCall: ResponseFunctionToolCall, ): CommandReviewDetails | undefined { @@ -57,12 +54,9 @@ export function parseToolCall( const { cmd } = toolCallArgs; const cmdReadableText = formatCommandForDisplay(cmd); - const autoApproval = computeAutoApproval(cmd); - return { cmd, cmdReadableText, - autoApproval, }; } @@ -109,87 +103,3 @@ function toStringArray(obj: unknown): Array | undefined { return undefined; } } - -// ---------------- safe‑command helpers ---------------- - -/** - * Attempts to determine whether `cmd` is composed exclusively of safe - * sub‑commands combined using only operators from the SAFE_SHELL_OPERATORS - * allow‑list. Returns the `SafeCommandReason` (taken from the first sub‑command) - * if the whole expression is safe; otherwise returns `null`. - */ -function computeAutoApproval(cmd: Array): SafeCommandReason | null { - // Fast path: a simple command with no shell processing. - const direct = isSafeCommand(cmd); - if (direct != null) { - return direct; - } - - // For expressions like ["bash", "-lc", "ls && pwd"] break down the inner - // string and verify each segment. - if ( - cmd.length === 3 && - cmd[0] === "bash" && - cmd[1] === "-lc" && - typeof cmd[2] === "string" - ) { - const parsed = parse(cmd[2], process.env ?? {}); - if (parsed.length === 0) { - return null; - } - - let current: Array = []; - let first: SafeCommandReason | null = null; - - const flush = (): boolean => { - if (current.length === 0) { - return true; - } - const safe = isSafeCommand(current); - if (safe == null) { - return false; - } - if (!first) { - first = safe; - } - current = []; - return true; - }; - - for (const part of parsed) { - if (typeof part === "string") { - // Simple word/argument token. - if (part === "(" || part === ")" || part === "{" || part === "}") { - // We treat explicit grouping tokens as unsafe because their - // semantics depend on the shell evaluation environment. - return null; - } - current.push(part); - } else if (part && typeof part === "object") { - const opToken = part as { op?: string }; - if (typeof opToken.op === "string") { - if (!flush()) { - return null; - } - if (!SAFE_SHELL_OPERATORS.has(opToken.op)) { - return null; - } - } else { - // Unknown object token kind (e.g. redirection) – treat as unsafe. - return null; - } - } else { - // Token types such as numbers / booleans are unexpected – treat as unsafe. - return null; - } - } - - if (!flush()) { - return null; - } - - return first; - } - - return null; -} From 7199511a34c1e11006263b5fca9b9cfb8a4fc172 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 15:16:51 -0700 Subject: [PATCH 07/84] Removes computeAutoApproval() and tightens up canAutoApprove() as the source of truth Signed-off-by: Michael Bolin --- codex-cli/src/approvals.ts | 127 ++++++++++-------- .../components/chat/use-message-grouping.ts | 72 ---------- codex-cli/src/utils/agent/review.ts | 8 -- codex-cli/src/utils/parsers.ts | 112 ++------------- 4 files changed, 80 insertions(+), 239 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index 0cf3703b54..8a670b01ca 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -75,19 +75,72 @@ export function canAutoApprove( writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { - try { - if (command[0] === "apply_patch") { - return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) - : { - type: "reject", - reason: "Invalid apply_patch command", - }; + if (command[0] === "apply_patch") { + return command.length === 2 && typeof command[1] === "string" + ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + : { + type: "reject", + reason: "Invalid apply_patch command", + }; + } + + const isSafe = isSafeCommand(command); + if (isSafe != null) { + const { reason, group } = isSafe; + return { + type: "auto-approve", + reason, + group, + runInSandbox: false, + }; + } + + if ( + command[0] === "bash" && + command[1] === "-lc" && + typeof command[2] === "string" && + command.length === 3 + ) { + const applyPatchArg = tryParseApplyPatch(command[2]); + if (applyPatchArg != null) { + return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); } - const isSafe = isSafeCommand(command); - if (isSafe != null) { - const { reason, group } = isSafe; + let bashCmd; + try { + bashCmd = parse(command[2], env); + } catch (e) { + // In practice, there seem to be syntactically valid shell commands that + // shell-quote cannot parse, so we should not reject, but ask the user. + switch (policy) { + case "full-auto": + // In full-auto, we still run the command automatically, but must + // restrict it to the sandbox. + return { + type: "auto-approve", + reason: "Full auto mode", + group: "Running commands", + runInSandbox: true, + }; + case "suggest": + case "auto-edit": + // In all other modes, since we cannot reason about the command, we + // should ask the user. + return { + type: "ask-user", + }; + } + } + + // bashCmd could be a mix of strings and operators, e.g.: + // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] + // We try to ensure that *every* command segment is deemed safe and that + // all operators belong to an allow‑list. If so, the entire expression is + // considered auto‑approvable. + + const shellSafe = isEntireShellExpressionSafe(bashCmd); + if (shellSafe != null) { + const { reason, group } = shellSafe; return { type: "auto-approve", reason, @@ -95,58 +148,16 @@ export function canAutoApprove( runInSandbox: false, }; } + } - if ( - command[0] === "bash" && - command[1] === "-lc" && - typeof command[2] === "string" && - command.length === 3 - ) { - const applyPatchArg = tryParseApplyPatch(command[2]); - if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); - } - - const bashCmd = parse(command[2], env); - - // bashCmd could be a mix of strings and operators, e.g.: - // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] - // We try to ensure that *every* command segment is deemed safe and that - // all operators belong to an allow‑list. If so, the entire expression is - // considered auto‑approvable. - - const shellSafe = isEntireShellExpressionSafe(bashCmd); - if (shellSafe != null) { - const { reason, group } = shellSafe; - return { - type: "auto-approve", - reason, - group, - runInSandbox: false, - }; - } - } - - return policy === "full-auto" - ? { - type: "auto-approve", - reason: "Full auto mode", - group: "Running commands", - runInSandbox: true, - } - : { type: "ask-user" }; - } catch (err) { - if (policy === "full-auto") { - return { + return policy === "full-auto" + ? { type: "auto-approve", reason: "Full auto mode", group: "Running commands", runInSandbox: true, - }; - } else { - return { type: "ask-user" }; - } - } + } + : { type: "ask-user" }; } function canAutoApproveApplyPatch( diff --git a/codex-cli/src/components/chat/use-message-grouping.ts b/codex-cli/src/components/chat/use-message-grouping.ts index 75e51ac682..1e526821d0 100644 --- a/codex-cli/src/components/chat/use-message-grouping.ts +++ b/codex-cli/src/components/chat/use-message-grouping.ts @@ -1,8 +1,5 @@ import type { ResponseItem } from "openai/resources/responses/responses.mjs"; -import { parseToolCall } from "../../utils/parsers.js"; -import { useMemo } from "react"; - /** * Represents a grouped sequence of response items (e.g., function call batches). */ @@ -10,72 +7,3 @@ export type GroupedResponseItem = { label: string; items: Array; }; - -/** - * Custom hook to group recent response items for display batching. - * Returns counts of auto-approved tool call groups, the latest batch, - * and the count of user messages in the visible window. - */ -export function useMessageGrouping(visibleItems: Array): { - groupCounts: Record; - batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }>; - userMsgCount: number; -} { - return useMemo(() => { - // The grouping logic only depends on the subset of messages that are - // currently rendered (visibleItems). Using that as the sole dependency - // keeps recomputations to a minimum and avoids unnecessary work when the - // full list of `items` changes outside of the visible window. - let userMsgCount = 0; - const groupCounts: Record = {}; - visibleItems.forEach((m) => { - if (m.type === "function_call") { - const toolCall = parseToolCall(m); - if (toolCall?.autoApproval) { - const group = toolCall.autoApproval.group; - groupCounts[group] = (groupCounts[group] || 0) + 1; - } - } - if (m.type === "message" && m.role === "user") { - userMsgCount++; - } - }); - const lastFew = visibleItems.slice(-3); - const batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }> = - []; - if (lastFew[0]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[0]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew, - }, - }); - if (lastFew[2]?.type === "message") { - batch.push({ item: lastFew[2] }); - } - } else if (lastFew[1]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[1]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew.slice(1), - }, - }); - } else if (lastFew[2]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[2]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: [lastFew[2]], - }, - }); - } else { - lastFew.forEach((item) => batch.push({ item })); - } - return { groupCounts, batch, userMsgCount }; - // `items` is stable across renders while `visibleItems` changes based on - // the scroll window. Including only `visibleItems` avoids unnecessary - // recomputations while still producing correct results. - }, [visibleItems]); -} diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index a370388569..9a5e66a4ff 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,11 +1,3 @@ -import type { SafeCommandReason } from "../../approvals"; - -export type CommandReviewDetails = { - cmd: Array; - cmdReadableText: string; - autoApproval: SafeCommandReason | null; -}; - export enum ReviewDecision { YES = "yes", NO_CONTINUE = "no-continue", diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index 815e7b2f00..4461379c02 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -1,30 +1,16 @@ -import type { CommandReviewDetails } from "./agent/review.js"; import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; -import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; -import process from "process"; -import { parse } from "shell-quote"; import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is // required. -// Allowed shell operators that we consider "safe" as they do not introduce -// side‑effects on their own (unlike redirections). Parentheses and braces for -// grouping are excluded for simplicity. -const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ - "&&", - "||", - "|", - ";", -]); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -46,6 +32,17 @@ export function parseToolCallOutput(toolCallOutput: string): { } } +export type CommandReviewDetails = { + cmd: Array; + cmdReadableText: string; +}; + +/** + * Tries to parse a tool call and, if successful, returns an object that has + * both: + * - an array of strings to use with `ExecInput` and `canAutoApprove()` + * - a human-readable string to display to the user + */ export function parseToolCall( toolCall: ResponseFunctionToolCall, ): CommandReviewDetails | undefined { @@ -57,12 +54,9 @@ export function parseToolCall( const { cmd } = toolCallArgs; const cmdReadableText = formatCommandForDisplay(cmd); - const autoApproval = computeAutoApproval(cmd); - return { cmd, cmdReadableText, - autoApproval, }; } @@ -109,87 +103,3 @@ function toStringArray(obj: unknown): Array | undefined { return undefined; } } - -// ---------------- safe‑command helpers ---------------- - -/** - * Attempts to determine whether `cmd` is composed exclusively of safe - * sub‑commands combined using only operators from the SAFE_SHELL_OPERATORS - * allow‑list. Returns the `SafeCommandReason` (taken from the first sub‑command) - * if the whole expression is safe; otherwise returns `null`. - */ -function computeAutoApproval(cmd: Array): SafeCommandReason | null { - // Fast path: a simple command with no shell processing. - const direct = isSafeCommand(cmd); - if (direct != null) { - return direct; - } - - // For expressions like ["bash", "-lc", "ls && pwd"] break down the inner - // string and verify each segment. - if ( - cmd.length === 3 && - cmd[0] === "bash" && - cmd[1] === "-lc" && - typeof cmd[2] === "string" - ) { - const parsed = parse(cmd[2], process.env ?? {}); - if (parsed.length === 0) { - return null; - } - - let current: Array = []; - let first: SafeCommandReason | null = null; - - const flush = (): boolean => { - if (current.length === 0) { - return true; - } - const safe = isSafeCommand(current); - if (safe == null) { - return false; - } - if (!first) { - first = safe; - } - current = []; - return true; - }; - - for (const part of parsed) { - if (typeof part === "string") { - // Simple word/argument token. - if (part === "(" || part === ")" || part === "{" || part === "}") { - // We treat explicit grouping tokens as unsafe because their - // semantics depend on the shell evaluation environment. - return null; - } - current.push(part); - } else if (part && typeof part === "object") { - const opToken = part as { op?: string }; - if (typeof opToken.op === "string") { - if (!flush()) { - return null; - } - if (!SAFE_SHELL_OPERATORS.has(opToken.op)) { - return null; - } - } else { - // Unknown object token kind (e.g. redirection) – treat as unsafe. - return null; - } - } else { - // Token types such as numbers / booleans are unexpected – treat as unsafe. - return null; - } - } - - if (!flush()) { - return null; - } - - return first; - } - - return null; -} From 08d209f8fa49242bfa7979b3861ff6302550b6f3 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 16 Apr 2025 15:36:25 -0700 Subject: [PATCH 08/84] Removes computeAutoApproval() and tightens up canAutoApprove() as the source of truth Signed-off-by: Michael Bolin --- codex-cli/src/approvals.ts | 127 ++++++++++-------- .../components/chat/use-message-grouping.ts | 72 ---------- codex-cli/src/utils/agent/review.ts | 8 -- codex-cli/src/utils/parsers.ts | 112 ++------------- 4 files changed, 80 insertions(+), 239 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index 0cf3703b54..8a670b01ca 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -75,19 +75,72 @@ export function canAutoApprove( writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { - try { - if (command[0] === "apply_patch") { - return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) - : { - type: "reject", - reason: "Invalid apply_patch command", - }; + if (command[0] === "apply_patch") { + return command.length === 2 && typeof command[1] === "string" + ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + : { + type: "reject", + reason: "Invalid apply_patch command", + }; + } + + const isSafe = isSafeCommand(command); + if (isSafe != null) { + const { reason, group } = isSafe; + return { + type: "auto-approve", + reason, + group, + runInSandbox: false, + }; + } + + if ( + command[0] === "bash" && + command[1] === "-lc" && + typeof command[2] === "string" && + command.length === 3 + ) { + const applyPatchArg = tryParseApplyPatch(command[2]); + if (applyPatchArg != null) { + return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); } - const isSafe = isSafeCommand(command); - if (isSafe != null) { - const { reason, group } = isSafe; + let bashCmd; + try { + bashCmd = parse(command[2], env); + } catch (e) { + // In practice, there seem to be syntactically valid shell commands that + // shell-quote cannot parse, so we should not reject, but ask the user. + switch (policy) { + case "full-auto": + // In full-auto, we still run the command automatically, but must + // restrict it to the sandbox. + return { + type: "auto-approve", + reason: "Full auto mode", + group: "Running commands", + runInSandbox: true, + }; + case "suggest": + case "auto-edit": + // In all other modes, since we cannot reason about the command, we + // should ask the user. + return { + type: "ask-user", + }; + } + } + + // bashCmd could be a mix of strings and operators, e.g.: + // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] + // We try to ensure that *every* command segment is deemed safe and that + // all operators belong to an allow‑list. If so, the entire expression is + // considered auto‑approvable. + + const shellSafe = isEntireShellExpressionSafe(bashCmd); + if (shellSafe != null) { + const { reason, group } = shellSafe; return { type: "auto-approve", reason, @@ -95,58 +148,16 @@ export function canAutoApprove( runInSandbox: false, }; } + } - if ( - command[0] === "bash" && - command[1] === "-lc" && - typeof command[2] === "string" && - command.length === 3 - ) { - const applyPatchArg = tryParseApplyPatch(command[2]); - if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); - } - - const bashCmd = parse(command[2], env); - - // bashCmd could be a mix of strings and operators, e.g.: - // "ls || (true && pwd)" => [ 'ls', { op: '||' }, '(', 'true', { op: '&&' }, 'pwd', ')' ] - // We try to ensure that *every* command segment is deemed safe and that - // all operators belong to an allow‑list. If so, the entire expression is - // considered auto‑approvable. - - const shellSafe = isEntireShellExpressionSafe(bashCmd); - if (shellSafe != null) { - const { reason, group } = shellSafe; - return { - type: "auto-approve", - reason, - group, - runInSandbox: false, - }; - } - } - - return policy === "full-auto" - ? { - type: "auto-approve", - reason: "Full auto mode", - group: "Running commands", - runInSandbox: true, - } - : { type: "ask-user" }; - } catch (err) { - if (policy === "full-auto") { - return { + return policy === "full-auto" + ? { type: "auto-approve", reason: "Full auto mode", group: "Running commands", runInSandbox: true, - }; - } else { - return { type: "ask-user" }; - } - } + } + : { type: "ask-user" }; } function canAutoApproveApplyPatch( diff --git a/codex-cli/src/components/chat/use-message-grouping.ts b/codex-cli/src/components/chat/use-message-grouping.ts index 75e51ac682..1e526821d0 100644 --- a/codex-cli/src/components/chat/use-message-grouping.ts +++ b/codex-cli/src/components/chat/use-message-grouping.ts @@ -1,8 +1,5 @@ import type { ResponseItem } from "openai/resources/responses/responses.mjs"; -import { parseToolCall } from "../../utils/parsers.js"; -import { useMemo } from "react"; - /** * Represents a grouped sequence of response items (e.g., function call batches). */ @@ -10,72 +7,3 @@ export type GroupedResponseItem = { label: string; items: Array; }; - -/** - * Custom hook to group recent response items for display batching. - * Returns counts of auto-approved tool call groups, the latest batch, - * and the count of user messages in the visible window. - */ -export function useMessageGrouping(visibleItems: Array): { - groupCounts: Record; - batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }>; - userMsgCount: number; -} { - return useMemo(() => { - // The grouping logic only depends on the subset of messages that are - // currently rendered (visibleItems). Using that as the sole dependency - // keeps recomputations to a minimum and avoids unnecessary work when the - // full list of `items` changes outside of the visible window. - let userMsgCount = 0; - const groupCounts: Record = {}; - visibleItems.forEach((m) => { - if (m.type === "function_call") { - const toolCall = parseToolCall(m); - if (toolCall?.autoApproval) { - const group = toolCall.autoApproval.group; - groupCounts[group] = (groupCounts[group] || 0) + 1; - } - } - if (m.type === "message" && m.role === "user") { - userMsgCount++; - } - }); - const lastFew = visibleItems.slice(-3); - const batch: Array<{ item?: ResponseItem; group?: GroupedResponseItem }> = - []; - if (lastFew[0]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[0]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew, - }, - }); - if (lastFew[2]?.type === "message") { - batch.push({ item: lastFew[2] }); - } - } else if (lastFew[1]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[1]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: lastFew.slice(1), - }, - }); - } else if (lastFew[2]?.type === "function_call") { - const toolCall = parseToolCall(lastFew[2]); - batch.push({ - group: { - label: toolCall?.autoApproval?.group || "Running command", - items: [lastFew[2]], - }, - }); - } else { - lastFew.forEach((item) => batch.push({ item })); - } - return { groupCounts, batch, userMsgCount }; - // `items` is stable across renders while `visibleItems` changes based on - // the scroll window. Including only `visibleItems` avoids unnecessary - // recomputations while still producing correct results. - }, [visibleItems]); -} diff --git a/codex-cli/src/utils/agent/review.ts b/codex-cli/src/utils/agent/review.ts index a370388569..9a5e66a4ff 100644 --- a/codex-cli/src/utils/agent/review.ts +++ b/codex-cli/src/utils/agent/review.ts @@ -1,11 +1,3 @@ -import type { SafeCommandReason } from "../../approvals"; - -export type CommandReviewDetails = { - cmd: Array; - cmdReadableText: string; - autoApproval: SafeCommandReason | null; -}; - export enum ReviewDecision { YES = "yes", NO_CONTINUE = "no-continue", diff --git a/codex-cli/src/utils/parsers.ts b/codex-cli/src/utils/parsers.ts index 815e7b2f00..4461379c02 100644 --- a/codex-cli/src/utils/parsers.ts +++ b/codex-cli/src/utils/parsers.ts @@ -1,30 +1,16 @@ -import type { CommandReviewDetails } from "./agent/review.js"; import type { ExecInput, ExecOutputMetadata, } from "./agent/sandbox/interface.js"; import type { ResponseFunctionToolCall } from "openai/resources/responses/responses.mjs"; -import { isSafeCommand, type SafeCommandReason } from "../approvals.js"; import { log } from "node:console"; -import process from "process"; -import { parse } from "shell-quote"; import { formatCommandForDisplay } from "src/format-command.js"; // The console utility import is intentionally explicit to avoid bundlers from // including the entire `console` module when only the `log` function is // required. -// Allowed shell operators that we consider "safe" as they do not introduce -// side‑effects on their own (unlike redirections). Parentheses and braces for -// grouping are excluded for simplicity. -const SAFE_SHELL_OPERATORS: ReadonlySet = new Set([ - "&&", - "||", - "|", - ";", -]); - export function parseToolCallOutput(toolCallOutput: string): { output: string; metadata: ExecOutputMetadata; @@ -46,6 +32,17 @@ export function parseToolCallOutput(toolCallOutput: string): { } } +export type CommandReviewDetails = { + cmd: Array; + cmdReadableText: string; +}; + +/** + * Tries to parse a tool call and, if successful, returns an object that has + * both: + * - an array of strings to use with `ExecInput` and `canAutoApprove()` + * - a human-readable string to display to the user + */ export function parseToolCall( toolCall: ResponseFunctionToolCall, ): CommandReviewDetails | undefined { @@ -57,12 +54,9 @@ export function parseToolCall( const { cmd } = toolCallArgs; const cmdReadableText = formatCommandForDisplay(cmd); - const autoApproval = computeAutoApproval(cmd); - return { cmd, cmdReadableText, - autoApproval, }; } @@ -109,87 +103,3 @@ function toStringArray(obj: unknown): Array | undefined { return undefined; } } - -// ---------------- safe‑command helpers ---------------- - -/** - * Attempts to determine whether `cmd` is composed exclusively of safe - * sub‑commands combined using only operators from the SAFE_SHELL_OPERATORS - * allow‑list. Returns the `SafeCommandReason` (taken from the first sub‑command) - * if the whole expression is safe; otherwise returns `null`. - */ -function computeAutoApproval(cmd: Array): SafeCommandReason | null { - // Fast path: a simple command with no shell processing. - const direct = isSafeCommand(cmd); - if (direct != null) { - return direct; - } - - // For expressions like ["bash", "-lc", "ls && pwd"] break down the inner - // string and verify each segment. - if ( - cmd.length === 3 && - cmd[0] === "bash" && - cmd[1] === "-lc" && - typeof cmd[2] === "string" - ) { - const parsed = parse(cmd[2], process.env ?? {}); - if (parsed.length === 0) { - return null; - } - - let current: Array = []; - let first: SafeCommandReason | null = null; - - const flush = (): boolean => { - if (current.length === 0) { - return true; - } - const safe = isSafeCommand(current); - if (safe == null) { - return false; - } - if (!first) { - first = safe; - } - current = []; - return true; - }; - - for (const part of parsed) { - if (typeof part === "string") { - // Simple word/argument token. - if (part === "(" || part === ")" || part === "{" || part === "}") { - // We treat explicit grouping tokens as unsafe because their - // semantics depend on the shell evaluation environment. - return null; - } - current.push(part); - } else if (part && typeof part === "object") { - const opToken = part as { op?: string }; - if (typeof opToken.op === "string") { - if (!flush()) { - return null; - } - if (!SAFE_SHELL_OPERATORS.has(opToken.op)) { - return null; - } - } else { - // Unknown object token kind (e.g. redirection) – treat as unsafe. - return null; - } - } else { - // Token types such as numbers / booleans are unexpected – treat as unsafe. - return null; - } - } - - if (!flush()) { - return null; - } - - return first; - } - - return null; -} From 832b98a5577d08106f242668faea4fd9c3192f5e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 09:52:49 -0700 Subject: [PATCH 09/84] use spawn instead of exec to avoid injection vulnerability --- .../src/components/chat/terminal-chat.tsx | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index 298d9208e2..f228ed19c4 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -29,7 +29,7 @@ import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; import { Box, Text } from "ink"; -import { exec } from "node:child_process"; +import { spawn } from "node:child_process"; import OpenAI from "openai"; import React, { useEffect, useMemo, useRef, useState } from "react"; import { inspect } from "util"; @@ -300,15 +300,10 @@ export default function TerminalChat({ agentRef.current = undefined; forceUpdate(); // re‑render after teardown too }; - // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps - // so switching modes or showing confirmation dialogs doesn’t tear down the loop. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - model, - config, - requestConfirmation, - additionalWritableRoots, - ]); + // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps + // so switching modes or showing confirmation dialogs doesn’t tear down the loop. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [model, config, requestConfirmation, additionalWritableRoots]); // whenever loading starts/stops, reset or start a timer — but pause the // timer while a confirmation overlay is displayed so we don't trigger a @@ -369,9 +364,10 @@ export default function TerminalChat({ const safePreview = preview.replace(/"/g, '\\"'); const title = "Codex CLI"; const cwd = PWD; - exec( - `osascript -e 'display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"'`, - ); + spawn("osascript", [ + "-e", + `display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"`, + ]); } } } @@ -597,7 +593,11 @@ export default function TerminalChat({ setApprovalPolicy(newMode as ApprovalPolicy); // update existing AgentLoop instance if (agentRef.current) { - (agentRef.current as unknown as { approvalPolicy: ApprovalPolicy }).approvalPolicy = newMode as ApprovalPolicy; + ( + agentRef.current as unknown as { + approvalPolicy: ApprovalPolicy; + } + ).approvalPolicy = newMode as ApprovalPolicy; } setItems((prev) => [ ...prev, From 66ae9fd234f3d5448ec74a8a381e67736f61af44 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:01:18 -0700 Subject: [PATCH 10/84] re-enable Prettier check for codex-cli in CI --- .github/workflows/ci.yml | 6 +- .../chat/terminal-chat-command-review.tsx | 79 ++++++++++--------- codex-cli/tests/input-utils.test.ts | 10 ++- 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 695190c4c9..378aea50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,11 @@ jobs: # Run all tasks using workspace filters - - name: Check formatting + - name: Check TypeScript code formatting. + working-directory: codex-cli + run: pnpm run format + + - name: Check Markdown and config file formatting. run: pnpm run format - name: Run tests diff --git a/codex-cli/src/components/chat/terminal-chat-command-review.tsx b/codex-cli/src/components/chat/terminal-chat-command-review.tsx index eadb9071ae..912af97961 100644 --- a/codex-cli/src/components/chat/terminal-chat-command-review.tsx +++ b/codex-cli/src/components/chat/terminal-chat-command-review.tsx @@ -121,46 +121,47 @@ export function TerminalChatCommandReview({ useInput( (input, key) => { - if (mode === "select") { - if (input === "y") { - onReviewCommand(ReviewDecision.YES); - } else if (input === "x") { - onReviewCommand(ReviewDecision.EXPLAIN); - } else if (input === "e") { - setMode("input"); - } else if (input === "n") { - onReviewCommand( - ReviewDecision.NO_CONTINUE, - "Don't do that, keep going though", - ); - } else if (input === "a" && showAlwaysApprove) { - onReviewCommand(ReviewDecision.ALWAYS); - } else if (input === "s") { - // switch approval mode - onSwitchApprovalMode(); - } else if (key.escape) { - onReviewCommand(ReviewDecision.NO_EXIT); + if (mode === "select") { + if (input === "y") { + onReviewCommand(ReviewDecision.YES); + } else if (input === "x") { + onReviewCommand(ReviewDecision.EXPLAIN); + } else if (input === "e") { + setMode("input"); + } else if (input === "n") { + onReviewCommand( + ReviewDecision.NO_CONTINUE, + "Don't do that, keep going though", + ); + } else if (input === "a" && showAlwaysApprove) { + onReviewCommand(ReviewDecision.ALWAYS); + } else if (input === "s") { + // switch approval mode + onSwitchApprovalMode(); + } else if (key.escape) { + onReviewCommand(ReviewDecision.NO_EXIT); + } + } else if (mode === "explanation") { + // When in explanation mode, any key returns to select mode + if (key.return || key.escape || input === "x") { + setMode("select"); + } + } else { + // text entry mode + if (key.return) { + // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE + const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; + onReviewCommand(ReviewDecision.NO_CONTINUE, custom); + } else if (key.escape) { + // treat escape as denial with default message as well + onReviewCommand( + ReviewDecision.NO_CONTINUE, + msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, + ); + } } - } else if (mode === "explanation") { - // When in explanation mode, any key returns to select mode - if (key.return || key.escape || input === "x") { - setMode("select"); - } - } else { - // text entry mode - if (key.return) { - // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE - const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; - onReviewCommand(ReviewDecision.NO_CONTINUE, custom); - } else if (key.escape) { - // treat escape as denial with default message as well - onReviewCommand( - ReviewDecision.NO_CONTINUE, - msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, - ); - } - } - }, { isActive } + }, + { isActive }, ); return ( diff --git a/codex-cli/tests/input-utils.test.ts b/codex-cli/tests/input-utils.test.ts index 5290e55488..780a384240 100644 --- a/codex-cli/tests/input-utils.test.ts +++ b/codex-cli/tests/input-utils.test.ts @@ -14,9 +14,13 @@ describe("createInputItem", () => { it("includes image content for existing file", async () => { const fakeBuffer = Buffer.from("fake image content"); - const readSpy = vi.spyOn(fs, "readFile").mockResolvedValue(fakeBuffer as any); + const readSpy = vi + .spyOn(fs, "readFile") + .mockResolvedValue(fakeBuffer as any); const result = await createInputItem("hello", ["dummy-path"]); - const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString("base64")}`; + const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString( + "base64", + )}`; expect(result.role).toBe("user"); expect(result.type).toBe("message"); expect(result.content.length).toBe(2); @@ -40,4 +44,4 @@ describe("createInputItem", () => { text: "[missing image: does-not-exist.png]", }); }); -}); \ No newline at end of file +}); From ede10c12476349cf070b07aeecb239db7e81d646 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:01:41 -0700 Subject: [PATCH 11/84] re-enable Prettier check for codex-cli in CI --- .github/workflows/ci.yml | 6 +- .../chat/terminal-chat-command-review.tsx | 79 ++++++++++--------- codex-cli/tests/input-utils.test.ts | 10 ++- 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 695190c4c9..378aea50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,11 @@ jobs: # Run all tasks using workspace filters - - name: Check formatting + - name: Check TypeScript code formatting. + working-directory: codex-cli + run: pnpm run format + + - name: Check Markdown and config file formatting. run: pnpm run format - name: Run tests diff --git a/codex-cli/src/components/chat/terminal-chat-command-review.tsx b/codex-cli/src/components/chat/terminal-chat-command-review.tsx index eadb9071ae..912af97961 100644 --- a/codex-cli/src/components/chat/terminal-chat-command-review.tsx +++ b/codex-cli/src/components/chat/terminal-chat-command-review.tsx @@ -121,46 +121,47 @@ export function TerminalChatCommandReview({ useInput( (input, key) => { - if (mode === "select") { - if (input === "y") { - onReviewCommand(ReviewDecision.YES); - } else if (input === "x") { - onReviewCommand(ReviewDecision.EXPLAIN); - } else if (input === "e") { - setMode("input"); - } else if (input === "n") { - onReviewCommand( - ReviewDecision.NO_CONTINUE, - "Don't do that, keep going though", - ); - } else if (input === "a" && showAlwaysApprove) { - onReviewCommand(ReviewDecision.ALWAYS); - } else if (input === "s") { - // switch approval mode - onSwitchApprovalMode(); - } else if (key.escape) { - onReviewCommand(ReviewDecision.NO_EXIT); + if (mode === "select") { + if (input === "y") { + onReviewCommand(ReviewDecision.YES); + } else if (input === "x") { + onReviewCommand(ReviewDecision.EXPLAIN); + } else if (input === "e") { + setMode("input"); + } else if (input === "n") { + onReviewCommand( + ReviewDecision.NO_CONTINUE, + "Don't do that, keep going though", + ); + } else if (input === "a" && showAlwaysApprove) { + onReviewCommand(ReviewDecision.ALWAYS); + } else if (input === "s") { + // switch approval mode + onSwitchApprovalMode(); + } else if (key.escape) { + onReviewCommand(ReviewDecision.NO_EXIT); + } + } else if (mode === "explanation") { + // When in explanation mode, any key returns to select mode + if (key.return || key.escape || input === "x") { + setMode("select"); + } + } else { + // text entry mode + if (key.return) { + // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE + const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; + onReviewCommand(ReviewDecision.NO_CONTINUE, custom); + } else if (key.escape) { + // treat escape as denial with default message as well + onReviewCommand( + ReviewDecision.NO_CONTINUE, + msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, + ); + } } - } else if (mode === "explanation") { - // When in explanation mode, any key returns to select mode - if (key.return || key.escape || input === "x") { - setMode("select"); - } - } else { - // text entry mode - if (key.return) { - // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE - const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; - onReviewCommand(ReviewDecision.NO_CONTINUE, custom); - } else if (key.escape) { - // treat escape as denial with default message as well - onReviewCommand( - ReviewDecision.NO_CONTINUE, - msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, - ); - } - } - }, { isActive } + }, + { isActive }, ); return ( diff --git a/codex-cli/tests/input-utils.test.ts b/codex-cli/tests/input-utils.test.ts index 5290e55488..780a384240 100644 --- a/codex-cli/tests/input-utils.test.ts +++ b/codex-cli/tests/input-utils.test.ts @@ -14,9 +14,13 @@ describe("createInputItem", () => { it("includes image content for existing file", async () => { const fakeBuffer = Buffer.from("fake image content"); - const readSpy = vi.spyOn(fs, "readFile").mockResolvedValue(fakeBuffer as any); + const readSpy = vi + .spyOn(fs, "readFile") + .mockResolvedValue(fakeBuffer as any); const result = await createInputItem("hello", ["dummy-path"]); - const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString("base64")}`; + const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString( + "base64", + )}`; expect(result.role).toBe("user"); expect(result.type).toBe("message"); expect(result.content.length).toBe(2); @@ -40,4 +44,4 @@ describe("createInputItem", () => { text: "[missing image: does-not-exist.png]", }); }); -}); \ No newline at end of file +}); From be9d216aabc0cb62e1d965b428996f50d81ca7db Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:01:41 -0700 Subject: [PATCH 12/84] re-enable Prettier check for codex-cli in CI --- .github/workflows/ci.yml | 6 +- .../chat/terminal-chat-command-review.tsx | 79 ++++++++++--------- .../src/components/chat/terminal-chat.tsx | 19 +++-- codex-cli/tests/input-utils.test.ts | 10 ++- 4 files changed, 61 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 695190c4c9..378aea50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,11 @@ jobs: # Run all tasks using workspace filters - - name: Check formatting + - name: Check TypeScript code formatting. + working-directory: codex-cli + run: pnpm run format + + - name: Check Markdown and config file formatting. run: pnpm run format - name: Run tests diff --git a/codex-cli/src/components/chat/terminal-chat-command-review.tsx b/codex-cli/src/components/chat/terminal-chat-command-review.tsx index eadb9071ae..912af97961 100644 --- a/codex-cli/src/components/chat/terminal-chat-command-review.tsx +++ b/codex-cli/src/components/chat/terminal-chat-command-review.tsx @@ -121,46 +121,47 @@ export function TerminalChatCommandReview({ useInput( (input, key) => { - if (mode === "select") { - if (input === "y") { - onReviewCommand(ReviewDecision.YES); - } else if (input === "x") { - onReviewCommand(ReviewDecision.EXPLAIN); - } else if (input === "e") { - setMode("input"); - } else if (input === "n") { - onReviewCommand( - ReviewDecision.NO_CONTINUE, - "Don't do that, keep going though", - ); - } else if (input === "a" && showAlwaysApprove) { - onReviewCommand(ReviewDecision.ALWAYS); - } else if (input === "s") { - // switch approval mode - onSwitchApprovalMode(); - } else if (key.escape) { - onReviewCommand(ReviewDecision.NO_EXIT); + if (mode === "select") { + if (input === "y") { + onReviewCommand(ReviewDecision.YES); + } else if (input === "x") { + onReviewCommand(ReviewDecision.EXPLAIN); + } else if (input === "e") { + setMode("input"); + } else if (input === "n") { + onReviewCommand( + ReviewDecision.NO_CONTINUE, + "Don't do that, keep going though", + ); + } else if (input === "a" && showAlwaysApprove) { + onReviewCommand(ReviewDecision.ALWAYS); + } else if (input === "s") { + // switch approval mode + onSwitchApprovalMode(); + } else if (key.escape) { + onReviewCommand(ReviewDecision.NO_EXIT); + } + } else if (mode === "explanation") { + // When in explanation mode, any key returns to select mode + if (key.return || key.escape || input === "x") { + setMode("select"); + } + } else { + // text entry mode + if (key.return) { + // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE + const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; + onReviewCommand(ReviewDecision.NO_CONTINUE, custom); + } else if (key.escape) { + // treat escape as denial with default message as well + onReviewCommand( + ReviewDecision.NO_CONTINUE, + msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, + ); + } } - } else if (mode === "explanation") { - // When in explanation mode, any key returns to select mode - if (key.return || key.escape || input === "x") { - setMode("select"); - } - } else { - // text entry mode - if (key.return) { - // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE - const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; - onReviewCommand(ReviewDecision.NO_CONTINUE, custom); - } else if (key.escape) { - // treat escape as denial with default message as well - onReviewCommand( - ReviewDecision.NO_CONTINUE, - msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, - ); - } - } - }, { isActive } + }, + { isActive }, ); return ( diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index 298d9208e2..e341cdfbbb 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -300,15 +300,10 @@ export default function TerminalChat({ agentRef.current = undefined; forceUpdate(); // re‑render after teardown too }; - // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps - // so switching modes or showing confirmation dialogs doesn’t tear down the loop. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - model, - config, - requestConfirmation, - additionalWritableRoots, - ]); + // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps + // so switching modes or showing confirmation dialogs doesn’t tear down the loop. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [model, config, requestConfirmation, additionalWritableRoots]); // whenever loading starts/stops, reset or start a timer — but pause the // timer while a confirmation overlay is displayed so we don't trigger a @@ -597,7 +592,11 @@ export default function TerminalChat({ setApprovalPolicy(newMode as ApprovalPolicy); // update existing AgentLoop instance if (agentRef.current) { - (agentRef.current as unknown as { approvalPolicy: ApprovalPolicy }).approvalPolicy = newMode as ApprovalPolicy; + ( + agentRef.current as unknown as { + approvalPolicy: ApprovalPolicy; + } + ).approvalPolicy = newMode as ApprovalPolicy; } setItems((prev) => [ ...prev, diff --git a/codex-cli/tests/input-utils.test.ts b/codex-cli/tests/input-utils.test.ts index 5290e55488..780a384240 100644 --- a/codex-cli/tests/input-utils.test.ts +++ b/codex-cli/tests/input-utils.test.ts @@ -14,9 +14,13 @@ describe("createInputItem", () => { it("includes image content for existing file", async () => { const fakeBuffer = Buffer.from("fake image content"); - const readSpy = vi.spyOn(fs, "readFile").mockResolvedValue(fakeBuffer as any); + const readSpy = vi + .spyOn(fs, "readFile") + .mockResolvedValue(fakeBuffer as any); const result = await createInputItem("hello", ["dummy-path"]); - const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString("base64")}`; + const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString( + "base64", + )}`; expect(result.role).toBe("user"); expect(result.type).toBe("message"); expect(result.content.length).toBe(2); @@ -40,4 +44,4 @@ describe("createInputItem", () => { text: "[missing image: does-not-exist.png]", }); }); -}); \ No newline at end of file +}); From 19a43ad15b679721d3c33aa01f4f04dd3bf2797c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:03:17 -0700 Subject: [PATCH 13/84] use spawn instead of exec to avoid injection vulnerability --- codex-cli/src/components/chat/terminal-chat.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index e341cdfbbb..f228ed19c4 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -29,7 +29,7 @@ import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; import { Box, Text } from "ink"; -import { exec } from "node:child_process"; +import { spawn } from "node:child_process"; import OpenAI from "openai"; import React, { useEffect, useMemo, useRef, useState } from "react"; import { inspect } from "util"; @@ -364,9 +364,10 @@ export default function TerminalChat({ const safePreview = preview.replace(/"/g, '\\"'); const title = "Codex CLI"; const cwd = PWD; - exec( - `osascript -e 'display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"'`, - ); + spawn("osascript", [ + "-e", + `display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"`, + ]); } } } From 3bbeba28ce0617a5b2d0e53eb96792fe93a5d3f6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:26:23 -0700 Subject: [PATCH 14/84] CONFIG_DIR should not be in the list of writable roots by default --- .../src/utils/agent/sandbox/macos-seatbelt.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 0317458261..760ba63d8c 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -3,11 +3,9 @@ import type { SpawnOptions } from "child_process"; import { exec } from "./raw-exec.js"; import { log } from "../log.js"; -import { CONFIG_DIR } from "src/utils/config.js"; function getCommonRoots() { return [ - CONFIG_DIR, // Without this root, it'll cause: // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable `${process.env["HOME"]}/.pyenv`, @@ -17,16 +15,17 @@ function getCommonRoots() { export function execWithSeatbelt( cmd: Array, opts: SpawnOptions, - writableRoots: Array, + writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { let scopedWritePolicy: string; let policyTemplateParams: Array; - if (writableRoots.length > 0) { - // Add `~/.codex` to the list of writable roots - // (if there's any already, not in read-only mode) - getCommonRoots().map((root) => writableRoots.push(root)); - const { policies, params } = writableRoots + + const fullWritableRoots = [...writableRoots, ...getCommonRoots()]; + // In practice, fullWritableRoots will be non-empty, but we check just in + // case the logic to build up fullWritableRoots changes. + if (fullWritableRoots.length > 0) { + const { policies, params } = fullWritableRoots .map((root, index) => ({ policy: `(subpath (param "WRITABLE_ROOT_${index}"))`, param: `-DWRITABLE_ROOT_${index}=${root}`, From 8a4968ae9fe5353076ed6d38244f4be16d032873 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:01:41 -0700 Subject: [PATCH 15/84] re-enable Prettier check for codex-cli in CI --- .github/workflows/ci.yml | 6 +- .../chat/terminal-chat-command-review.tsx | 79 ++++++++++--------- .../src/components/chat/terminal-chat.tsx | 19 +++-- codex-cli/tests/input-utils.test.ts | 10 ++- 4 files changed, 61 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 695190c4c9..e99c2daa04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,11 @@ jobs: # Run all tasks using workspace filters - - name: Check formatting + - name: Check TypeScript code formatting + working-directory: codex-cli + run: pnpm run format + + - name: Check Markdown and config file formatting run: pnpm run format - name: Run tests diff --git a/codex-cli/src/components/chat/terminal-chat-command-review.tsx b/codex-cli/src/components/chat/terminal-chat-command-review.tsx index eadb9071ae..912af97961 100644 --- a/codex-cli/src/components/chat/terminal-chat-command-review.tsx +++ b/codex-cli/src/components/chat/terminal-chat-command-review.tsx @@ -121,46 +121,47 @@ export function TerminalChatCommandReview({ useInput( (input, key) => { - if (mode === "select") { - if (input === "y") { - onReviewCommand(ReviewDecision.YES); - } else if (input === "x") { - onReviewCommand(ReviewDecision.EXPLAIN); - } else if (input === "e") { - setMode("input"); - } else if (input === "n") { - onReviewCommand( - ReviewDecision.NO_CONTINUE, - "Don't do that, keep going though", - ); - } else if (input === "a" && showAlwaysApprove) { - onReviewCommand(ReviewDecision.ALWAYS); - } else if (input === "s") { - // switch approval mode - onSwitchApprovalMode(); - } else if (key.escape) { - onReviewCommand(ReviewDecision.NO_EXIT); + if (mode === "select") { + if (input === "y") { + onReviewCommand(ReviewDecision.YES); + } else if (input === "x") { + onReviewCommand(ReviewDecision.EXPLAIN); + } else if (input === "e") { + setMode("input"); + } else if (input === "n") { + onReviewCommand( + ReviewDecision.NO_CONTINUE, + "Don't do that, keep going though", + ); + } else if (input === "a" && showAlwaysApprove) { + onReviewCommand(ReviewDecision.ALWAYS); + } else if (input === "s") { + // switch approval mode + onSwitchApprovalMode(); + } else if (key.escape) { + onReviewCommand(ReviewDecision.NO_EXIT); + } + } else if (mode === "explanation") { + // When in explanation mode, any key returns to select mode + if (key.return || key.escape || input === "x") { + setMode("select"); + } + } else { + // text entry mode + if (key.return) { + // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE + const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; + onReviewCommand(ReviewDecision.NO_CONTINUE, custom); + } else if (key.escape) { + // treat escape as denial with default message as well + onReviewCommand( + ReviewDecision.NO_CONTINUE, + msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, + ); + } } - } else if (mode === "explanation") { - // When in explanation mode, any key returns to select mode - if (key.return || key.escape || input === "x") { - setMode("select"); - } - } else { - // text entry mode - if (key.return) { - // if user hit enter on empty msg, fall back to DEFAULT_DENY_MESSAGE - const custom = msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg; - onReviewCommand(ReviewDecision.NO_CONTINUE, custom); - } else if (key.escape) { - // treat escape as denial with default message as well - onReviewCommand( - ReviewDecision.NO_CONTINUE, - msg.trim() === "" ? DEFAULT_DENY_MESSAGE : msg, - ); - } - } - }, { isActive } + }, + { isActive }, ); return ( diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index 298d9208e2..e341cdfbbb 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -300,15 +300,10 @@ export default function TerminalChat({ agentRef.current = undefined; forceUpdate(); // re‑render after teardown too }; - // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps - // so switching modes or showing confirmation dialogs doesn’t tear down the loop. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - model, - config, - requestConfirmation, - additionalWritableRoots, - ]); + // We intentionally omit 'approvalPolicy' and 'confirmationPrompt' from the deps + // so switching modes or showing confirmation dialogs doesn’t tear down the loop. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [model, config, requestConfirmation, additionalWritableRoots]); // whenever loading starts/stops, reset or start a timer — but pause the // timer while a confirmation overlay is displayed so we don't trigger a @@ -597,7 +592,11 @@ export default function TerminalChat({ setApprovalPolicy(newMode as ApprovalPolicy); // update existing AgentLoop instance if (agentRef.current) { - (agentRef.current as unknown as { approvalPolicy: ApprovalPolicy }).approvalPolicy = newMode as ApprovalPolicy; + ( + agentRef.current as unknown as { + approvalPolicy: ApprovalPolicy; + } + ).approvalPolicy = newMode as ApprovalPolicy; } setItems((prev) => [ ...prev, diff --git a/codex-cli/tests/input-utils.test.ts b/codex-cli/tests/input-utils.test.ts index 5290e55488..780a384240 100644 --- a/codex-cli/tests/input-utils.test.ts +++ b/codex-cli/tests/input-utils.test.ts @@ -14,9 +14,13 @@ describe("createInputItem", () => { it("includes image content for existing file", async () => { const fakeBuffer = Buffer.from("fake image content"); - const readSpy = vi.spyOn(fs, "readFile").mockResolvedValue(fakeBuffer as any); + const readSpy = vi + .spyOn(fs, "readFile") + .mockResolvedValue(fakeBuffer as any); const result = await createInputItem("hello", ["dummy-path"]); - const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString("base64")}`; + const expectedUrl = `data:application/octet-stream;base64,${fakeBuffer.toString( + "base64", + )}`; expect(result.role).toBe("user"); expect(result.type).toBe("message"); expect(result.content.length).toBe(2); @@ -40,4 +44,4 @@ describe("createInputItem", () => { text: "[missing image: does-not-exist.png]", }); }); -}); \ No newline at end of file +}); From 0430ab327b761c6f05bd1daf33e40a2bd86a7b2f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:29:18 -0700 Subject: [PATCH 16/84] use spawn instead of exec to avoid injection vulnerability --- codex-cli/src/components/chat/terminal-chat.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index e341cdfbbb..f228ed19c4 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -29,7 +29,7 @@ import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; import { Box, Text } from "ink"; -import { exec } from "node:child_process"; +import { spawn } from "node:child_process"; import OpenAI from "openai"; import React, { useEffect, useMemo, useRef, useState } from "react"; import { inspect } from "util"; @@ -364,9 +364,10 @@ export default function TerminalChat({ const safePreview = preview.replace(/"/g, '\\"'); const title = "Codex CLI"; const cwd = PWD; - exec( - `osascript -e 'display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"'`, - ); + spawn("osascript", [ + "-e", + `display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"`, + ]); } } } From 5333ea726f68d181f518c4dcdc649c16d0c42678 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 10:29:18 -0700 Subject: [PATCH 17/84] CONFIG_DIR should not be in the list of writable roots by default --- .../src/utils/agent/sandbox/macos-seatbelt.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 0317458261..760ba63d8c 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -3,11 +3,9 @@ import type { SpawnOptions } from "child_process"; import { exec } from "./raw-exec.js"; import { log } from "../log.js"; -import { CONFIG_DIR } from "src/utils/config.js"; function getCommonRoots() { return [ - CONFIG_DIR, // Without this root, it'll cause: // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable `${process.env["HOME"]}/.pyenv`, @@ -17,16 +15,17 @@ function getCommonRoots() { export function execWithSeatbelt( cmd: Array, opts: SpawnOptions, - writableRoots: Array, + writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { let scopedWritePolicy: string; let policyTemplateParams: Array; - if (writableRoots.length > 0) { - // Add `~/.codex` to the list of writable roots - // (if there's any already, not in read-only mode) - getCommonRoots().map((root) => writableRoots.push(root)); - const { policies, params } = writableRoots + + const fullWritableRoots = [...writableRoots, ...getCommonRoots()]; + // In practice, fullWritableRoots will be non-empty, but we check just in + // case the logic to build up fullWritableRoots changes. + if (fullWritableRoots.length > 0) { + const { policies, params } = fullWritableRoots .map((root, index) => ({ policy: `(subpath (param "WRITABLE_ROOT_${index}"))`, param: `-DWRITABLE_ROOT_${index}=${root}`, From 6ac0fd5fc10b5b02a6befdf3d364b103a45ba4ef Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:07:35 -0700 Subject: [PATCH 18/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 16 ++--- .../components/chat/terminal-chat-input.tsx | 16 ++--- .../chat/terminal-chat-new-input.tsx | 16 ++--- .../src/components/chat/terminal-chat.tsx | 62 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++--- 10 files changed, 94 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..fdaf0c0823 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; // removed unused isLoggingEnabled import import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,7 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log("raw stdin: received collapsed ESC ESC – starting confirmation timer"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +61,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index 59265221d4..9a0fdbdab9 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -683,11 +683,7 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log("raw stdin: received collapsed ESC ESC – starting confirmation timer"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -712,15 +708,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 9ceb4bbccc..5197083be8 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -496,11 +496,7 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log("raw stdin: received collapsed ESC ESC – starting confirmation timer"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -522,15 +518,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index f228ed19c4..538a3b3561 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -197,30 +197,24 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -288,14 +282,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -377,9 +367,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -504,11 +492,7 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log("TerminalChat: interruptAgent invoked – calling agent.cancel()"); agent.cancel(); setLoading(false); @@ -544,13 +528,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..15c37475d4 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean( + this.execAbortController, + )} generation=${this.generation}`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..f719cafdb1 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -28,13 +28,10 @@ export function exec( const adaptedCommand = adaptCommandForPlatform(command); if ( - isLoggingEnabled() && JSON.stringify(adaptedCommand) !== JSON.stringify(command) ) { log( - `Command adapted for platform: ${command.join( - " ", - )} -> ${adaptedCommand.join(" ")}`, + `Command adapted for platform: ${command.join(" ")} -> ${adaptedCommand.join(" ")}`, ); } @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From f77625ea881bcf5d51bee68409a62c01db18c6b7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:07:35 -0700 Subject: [PATCH 19/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 18 ++---- .../components/chat/terminal-chat-input.tsx | 18 ++---- .../chat/terminal-chat-new-input.tsx | 18 ++---- .../src/components/chat/terminal-chat.tsx | 64 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++-- 10 files changed, 102 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..c0b305ef14 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; // removed unused isLoggingEnabled import import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,9 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +63,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index 59265221d4..943637dba5 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -683,11 +683,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -712,15 +710,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 9ceb4bbccc..a4558c94be 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -496,11 +496,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -522,15 +520,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index f228ed19c4..9ed1568dd8 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -197,30 +197,24 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -288,14 +282,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -377,9 +367,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -504,11 +492,9 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); agent.cancel(); setLoading(false); @@ -544,13 +530,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..172ba67295 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean(this.execAbortController)} generation=${ + this.generation + }`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..f2ca03bf49 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -27,10 +27,7 @@ export function exec( // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) const adaptedCommand = adaptCommandForPlatform(command); - if ( - isLoggingEnabled() && - JSON.stringify(adaptedCommand) !== JSON.stringify(command) - ) { + if (JSON.stringify(adaptedCommand) !== JSON.stringify(command)) { log( `Command adapted for platform: ${command.join( " ", @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From bce7ab2feafe7c919f507051af02ec5d39247b01 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:07:35 -0700 Subject: [PATCH 20/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 18 ++--- .../components/chat/terminal-chat-input.tsx | 18 ++--- .../chat/terminal-chat-new-input.tsx | 18 ++--- .../src/components/chat/terminal-chat.tsx | 65 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++-- 10 files changed, 103 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..fdc8bd218c 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,9 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +63,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index 59265221d4..943637dba5 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -683,11 +683,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -712,15 +710,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 9ceb4bbccc..a4558c94be 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -496,11 +496,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -522,15 +520,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index f228ed19c4..7602721652 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -197,30 +197,25 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -288,14 +283,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -377,9 +368,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -504,11 +493,9 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); agent.cancel(); setLoading(false); @@ -544,13 +531,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..172ba67295 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean(this.execAbortController)} generation=${ + this.generation + }`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..f2ca03bf49 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -27,10 +27,7 @@ export function exec( // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) const adaptedCommand = adaptCommandForPlatform(command); - if ( - isLoggingEnabled() && - JSON.stringify(adaptedCommand) !== JSON.stringify(command) - ) { + if (JSON.stringify(adaptedCommand) !== JSON.stringify(command)) { log( `Command adapted for platform: ${command.join( " ", @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From 75919d78fc019981dcece69664eeafad009405d7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:23:58 -0700 Subject: [PATCH 21/84] CONFIG_DIR should not be in the list of writable roots by default --- .../src/utils/agent/sandbox/macos-seatbelt.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 0317458261..760ba63d8c 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -3,11 +3,9 @@ import type { SpawnOptions } from "child_process"; import { exec } from "./raw-exec.js"; import { log } from "../log.js"; -import { CONFIG_DIR } from "src/utils/config.js"; function getCommonRoots() { return [ - CONFIG_DIR, // Without this root, it'll cause: // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable `${process.env["HOME"]}/.pyenv`, @@ -17,16 +15,17 @@ function getCommonRoots() { export function execWithSeatbelt( cmd: Array, opts: SpawnOptions, - writableRoots: Array, + writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { let scopedWritePolicy: string; let policyTemplateParams: Array; - if (writableRoots.length > 0) { - // Add `~/.codex` to the list of writable roots - // (if there's any already, not in read-only mode) - getCommonRoots().map((root) => writableRoots.push(root)); - const { policies, params } = writableRoots + + const fullWritableRoots = [...writableRoots, ...getCommonRoots()]; + // In practice, fullWritableRoots will be non-empty, but we check just in + // case the logic to build up fullWritableRoots changes. + if (fullWritableRoots.length > 0) { + const { policies, params } = fullWritableRoots .map((root, index) => ({ policy: `(subpath (param "WRITABLE_ROOT_${index}"))`, param: `-DWRITABLE_ROOT_${index}=${root}`, From 7a16e239fc72822bcf1728c77df1569f56c6af6f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:23:58 -0700 Subject: [PATCH 22/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 18 ++--- .../components/chat/terminal-chat-input.tsx | 18 ++--- .../chat/terminal-chat-new-input.tsx | 18 ++--- .../src/components/chat/terminal-chat.tsx | 65 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++-- 10 files changed, 103 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..fdc8bd218c 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,9 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +63,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index 59265221d4..943637dba5 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -683,11 +683,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -712,15 +710,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 9ceb4bbccc..a4558c94be 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -496,11 +496,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -522,15 +520,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index f228ed19c4..7602721652 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -197,30 +197,25 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -288,14 +283,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -377,9 +368,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -504,11 +493,9 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); agent.cancel(); setLoading(false); @@ -544,13 +531,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..172ba67295 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean(this.execAbortController)} generation=${ + this.generation + }`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..f2ca03bf49 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -27,10 +27,7 @@ export function exec( // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) const adaptedCommand = adaptCommandForPlatform(command); - if ( - isLoggingEnabled() && - JSON.stringify(adaptedCommand) !== JSON.stringify(command) - ) { + if (JSON.stringify(adaptedCommand) !== JSON.stringify(command)) { log( `Command adapted for platform: ${command.join( " ", @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From 70138aed178b1ca3287d2a02d7d69c6b5148de41 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:23:58 -0700 Subject: [PATCH 23/84] use spawn instead of exec to avoid injection vulnerability --- codex-cli/src/components/chat/terminal-chat.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index e341cdfbbb..f228ed19c4 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -29,7 +29,7 @@ import HelpOverlay from "../help-overlay.js"; import HistoryOverlay from "../history-overlay.js"; import ModelOverlay from "../model-overlay.js"; import { Box, Text } from "ink"; -import { exec } from "node:child_process"; +import { spawn } from "node:child_process"; import OpenAI from "openai"; import React, { useEffect, useMemo, useRef, useState } from "react"; import { inspect } from "util"; @@ -364,9 +364,10 @@ export default function TerminalChat({ const safePreview = preview.replace(/"/g, '\\"'); const title = "Codex CLI"; const cwd = PWD; - exec( - `osascript -e 'display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"'`, - ); + spawn("osascript", [ + "-e", + `display notification "${safePreview}" with title "${title}" subtitle "${cwd}" sound name "Ping"`, + ]); } } } From c03bb964384d4e9a93bf5797b84f885b049def71 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:23:58 -0700 Subject: [PATCH 24/84] reduce max output of ExecResult --- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 130 +++++++++++++----- 1 file changed, 97 insertions(+), 33 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index f2ca03bf49..ce10eae149 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -12,7 +12,10 @@ import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; -const MAX_BUFFER = 1024 * 100; // 100 KB +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; /** * This function should never return a rejected promise: errors should be @@ -143,37 +146,17 @@ export function exec( // resolve the promise and translate the failure into a regular // ExecResult object so the rest of the agent loop can carry on gracefully. - const stdoutChunks: Array = []; - const stderrChunks: Array = []; - let numStdoutBytes = 0; - let numStderrBytes = 0; - let hitMaxStdout = false; - let hitMaxStderr = false; + // Collect stdout and stderr up to configured limits + const stdoutCollector = createTruncatingCollector(); + const stderrCollector = createTruncatingCollector(); return new Promise((resolve) => { - child.stdout?.on("data", (data: Buffer) => { - if (!hitMaxStdout) { - numStdoutBytes += data.length; - if (numStdoutBytes <= MAX_BUFFER) { - stdoutChunks.push(data); - } else { - hitMaxStdout = true; - } - } - }); - child.stderr?.on("data", (data: Buffer) => { - if (!hitMaxStderr) { - numStderrBytes += data.length; - if (numStderrBytes <= MAX_BUFFER) { - stderrChunks.push(data); - } else { - hitMaxStderr = true; - } - } - }); + stdoutCollector.attach(child.stdout!); + stderrCollector.attach(child.stderr!); + child.on("exit", (code, signal) => { - const stdout = Buffer.concat(stdoutChunks).toString("utf8"); - const stderr = Buffer.concat(stderrChunks).toString("utf8"); + const stdout = stdoutCollector.getString(); + const stderr = stderrCollector.getString(); // Map (code, signal) to an exit code. We expect exactly one of the two // values to be non-null, but we code defensively to handle the case where @@ -192,19 +175,100 @@ export function exec( log( `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, ); - resolve({ + + const execResult = { stdout, stderr, exitCode, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); child.on("error", (err) => { - resolve({ + const execResult = { stdout: "", stderr: String(err), exitCode: 1, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); }); } + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +function createTruncatingCollector( + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +) { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + return { + attach(stream?: NodeJS.ReadableStream) { + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + totalBytes += data.length; + for (let i = 0; i < data.length; i++) { + if (data[i] === 0x0a) { + totalLines++; + } + } + if (totalBytes <= byteLimit && totalLines <= lineLimit) { + chunks.push(data); + } else { + hitLimit = true; + } + }); + }, + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} + +/** + * Adds a truncation warnings to stdout and stderr, if appropriate. + */ +function addTruncationWarningsIfNecessary( + execResult: ExecResult, + hitMaxStdout: boolean, + hitMaxStderr: boolean, +): ExecResult { + if (!hitMaxStdout && !hitMaxStderr) { + return execResult; + } else { + const { stdout, stderr, exitCode } = execResult; + return { + stdout: hitMaxStdout + ? stdout + "\n\n[Output truncated: too many lines or bytes]" + : stdout, + stderr: hitMaxStderr + ? stderr + "\n\n[Output truncated: too many lines or bytes]" + : stderr, + exitCode, + }; + } +} From 507bc768bd25bf520be46ffc721a1a83a24153bd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:23:58 -0700 Subject: [PATCH 25/84] CONFIG_DIR should not be in the list of writable roots by default --- .../src/utils/agent/sandbox/macos-seatbelt.ts | 15 +++++++-------- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 0317458261..760ba63d8c 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -3,11 +3,9 @@ import type { SpawnOptions } from "child_process"; import { exec } from "./raw-exec.js"; import { log } from "../log.js"; -import { CONFIG_DIR } from "src/utils/config.js"; function getCommonRoots() { return [ - CONFIG_DIR, // Without this root, it'll cause: // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable `${process.env["HOME"]}/.pyenv`, @@ -17,16 +15,17 @@ function getCommonRoots() { export function execWithSeatbelt( cmd: Array, opts: SpawnOptions, - writableRoots: Array, + writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { let scopedWritePolicy: string; let policyTemplateParams: Array; - if (writableRoots.length > 0) { - // Add `~/.codex` to the list of writable roots - // (if there's any already, not in read-only mode) - getCommonRoots().map((root) => writableRoots.push(root)); - const { policies, params } = writableRoots + + const fullWritableRoots = [...writableRoots, ...getCommonRoots()]; + // In practice, fullWritableRoots will be non-empty, but we check just in + // case the logic to build up fullWritableRoots changes. + if (fullWritableRoots.length > 0) { + const { policies, params } = fullWritableRoots .map((root, index) => ({ policy: `(subpath (param "WRITABLE_ROOT_${index}"))`, param: `-DWRITABLE_ROOT_${index}=${root}`, diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..e7bcdb2a13 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -21,7 +21,7 @@ const MAX_BUFFER = 1024 * 100; // 100 KB export function exec( command: Array, options: SpawnOptions, - _writableRoots: Array, + _writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) From cdb7325192bf8d9a9b1fa20f8039b5fee92f2314 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:45:21 -0700 Subject: [PATCH 26/84] reduce max output of ExecResult --- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 130 +++++++++++++----- 1 file changed, 97 insertions(+), 33 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 35ae8a4a92..1b9bc1db25 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -12,7 +12,10 @@ import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; -const MAX_BUFFER = 1024 * 100; // 100 KB +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; /** * This function should never return a rejected promise: errors should be @@ -143,37 +146,17 @@ export function exec( // resolve the promise and translate the failure into a regular // ExecResult object so the rest of the agent loop can carry on gracefully. - const stdoutChunks: Array = []; - const stderrChunks: Array = []; - let numStdoutBytes = 0; - let numStderrBytes = 0; - let hitMaxStdout = false; - let hitMaxStderr = false; + // Collect stdout and stderr up to configured limits + const stdoutCollector = createTruncatingCollector(); + const stderrCollector = createTruncatingCollector(); return new Promise((resolve) => { - child.stdout?.on("data", (data: Buffer) => { - if (!hitMaxStdout) { - numStdoutBytes += data.length; - if (numStdoutBytes <= MAX_BUFFER) { - stdoutChunks.push(data); - } else { - hitMaxStdout = true; - } - } - }); - child.stderr?.on("data", (data: Buffer) => { - if (!hitMaxStderr) { - numStderrBytes += data.length; - if (numStderrBytes <= MAX_BUFFER) { - stderrChunks.push(data); - } else { - hitMaxStderr = true; - } - } - }); + stdoutCollector.attach(child.stdout!); + stderrCollector.attach(child.stderr!); + child.on("exit", (code, signal) => { - const stdout = Buffer.concat(stdoutChunks).toString("utf8"); - const stderr = Buffer.concat(stderrChunks).toString("utf8"); + const stdout = stdoutCollector.getString(); + const stderr = stderrCollector.getString(); // Map (code, signal) to an exit code. We expect exactly one of the two // values to be non-null, but we code defensively to handle the case where @@ -192,19 +175,100 @@ export function exec( log( `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, ); - resolve({ + + const execResult = { stdout, stderr, exitCode, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); child.on("error", (err) => { - resolve({ + const execResult = { stdout: "", stderr: String(err), exitCode: 1, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); }); } + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +function createTruncatingCollector( + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +) { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + return { + attach(stream?: NodeJS.ReadableStream) { + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + totalBytes += data.length; + for (let i = 0; i < data.length; i++) { + if (data[i] === 0x0a) { + totalLines++; + } + } + if (totalBytes <= byteLimit && totalLines <= lineLimit) { + chunks.push(data); + } else { + hitLimit = true; + } + }); + }, + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} + +/** + * Adds a truncation warnings to stdout and stderr, if appropriate. + */ +function addTruncationWarningsIfNecessary( + execResult: ExecResult, + hitMaxStdout: boolean, + hitMaxStderr: boolean, +): ExecResult { + if (!hitMaxStdout && !hitMaxStderr) { + return execResult; + } else { + const { stdout, stderr, exitCode } = execResult; + return { + stdout: hitMaxStdout + ? stdout + "\n\n[Output truncated: too many lines or bytes]" + : stdout, + stderr: hitMaxStderr + ? stderr + "\n\n[Output truncated: too many lines or bytes]" + : stderr, + exitCode, + }; + } +} From 1757ff3eb222f01b33b6b1889f2d1d8aabc8d8a7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:45:21 -0700 Subject: [PATCH 27/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 18 ++--- .../components/chat/terminal-chat-input.tsx | 18 ++--- .../chat/terminal-chat-new-input.tsx | 18 ++--- .../src/components/chat/terminal-chat.tsx | 65 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++-- 10 files changed, 103 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..fdc8bd218c 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,9 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +63,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index 59265221d4..943637dba5 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -683,11 +683,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -712,15 +710,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 9ceb4bbccc..a4558c94be 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -496,11 +496,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -522,15 +520,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index f228ed19c4..7602721652 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -197,30 +197,25 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -288,14 +283,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -377,9 +368,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -504,11 +493,9 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); agent.cancel(); setLoading(false); @@ -544,13 +531,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..172ba67295 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean(this.execAbortController)} generation=${ + this.generation + }`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index e7bcdb2a13..35ae8a4a92 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -27,10 +27,7 @@ export function exec( // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) const adaptedCommand = adaptCommandForPlatform(command); - if ( - isLoggingEnabled() && - JSON.stringify(adaptedCommand) !== JSON.stringify(command) - ) { + if (JSON.stringify(adaptedCommand) !== JSON.stringify(command)) { log( `Command adapted for platform: ${command.join( " ", @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From cdef7dfddcb4090a4042b1c03a33c101dca9cc5f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 11:45:21 -0700 Subject: [PATCH 28/84] reduce max output of ExecResult --- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 130 +++++++++++++----- 1 file changed, 96 insertions(+), 34 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 35ae8a4a92..22add83ed8 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -12,7 +12,10 @@ import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; -const MAX_BUFFER = 1024 * 100; // 100 KB +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; /** * This function should never return a rejected promise: errors should be @@ -143,37 +146,14 @@ export function exec( // resolve the promise and translate the failure into a regular // ExecResult object so the rest of the agent loop can carry on gracefully. - const stdoutChunks: Array = []; - const stderrChunks: Array = []; - let numStdoutBytes = 0; - let numStderrBytes = 0; - let hitMaxStdout = false; - let hitMaxStderr = false; - return new Promise((resolve) => { - child.stdout?.on("data", (data: Buffer) => { - if (!hitMaxStdout) { - numStdoutBytes += data.length; - if (numStdoutBytes <= MAX_BUFFER) { - stdoutChunks.push(data); - } else { - hitMaxStdout = true; - } - } - }); - child.stderr?.on("data", (data: Buffer) => { - if (!hitMaxStderr) { - numStderrBytes += data.length; - if (numStderrBytes <= MAX_BUFFER) { - stderrChunks.push(data); - } else { - hitMaxStderr = true; - } - } - }); + // Collect stdout and stderr up to configured limits. + const stdoutCollector = createTruncatingCollector(child.stdout!); + const stderrCollector = createTruncatingCollector(child.stderr!); + child.on("exit", (code, signal) => { - const stdout = Buffer.concat(stdoutChunks).toString("utf8"); - const stderr = Buffer.concat(stderrChunks).toString("utf8"); + const stdout = stdoutCollector.getString(); + const stderr = stderrCollector.getString(); // Map (code, signal) to an exit code. We expect exactly one of the two // values to be non-null, but we code defensively to handle the case where @@ -192,19 +172,101 @@ export function exec( log( `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, ); - resolve({ + + const execResult = { stdout, stderr, exitCode, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); child.on("error", (err) => { - resolve({ + const execResult = { stdout: "", stderr: String(err), exitCode: 1, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); }); } + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +function createTruncatingCollector( + stream: NodeJS.ReadableStream, + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +) { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + totalBytes += data.length; + for (let i = 0; i < data.length; i++) { + if (data[i] === 0x0a) { + totalLines++; + } + } + if (totalBytes <= byteLimit && totalLines <= lineLimit) { + chunks.push(data); + } else { + hitLimit = true; + } + }); + + return { + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} + +/** + * Adds a truncation warnings to stdout and stderr, if appropriate. + */ +function addTruncationWarningsIfNecessary( + execResult: ExecResult, + hitMaxStdout: boolean, + hitMaxStderr: boolean, +): ExecResult { + if (!hitMaxStdout && !hitMaxStderr) { + return execResult; + } else { + const { stdout, stderr, exitCode } = execResult; + return { + stdout: hitMaxStdout + ? stdout + "\n\n[Output truncated: too many lines or bytes]" + : stdout, + stderr: hitMaxStderr + ? stderr + "\n\n[Output truncated: too many lines or bytes]" + : stderr, + exitCode, + }; + } +} From a7a4a69ccc5527bf691b2f0d1d230745790f1cdd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 18:29:11 -0700 Subject: [PATCH 29/84] CONFIG_DIR should not be in the list of writable roots by default --- .../src/utils/agent/sandbox/macos-seatbelt.ts | 15 +++++++-------- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 0317458261..760ba63d8c 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -3,11 +3,9 @@ import type { SpawnOptions } from "child_process"; import { exec } from "./raw-exec.js"; import { log } from "../log.js"; -import { CONFIG_DIR } from "src/utils/config.js"; function getCommonRoots() { return [ - CONFIG_DIR, // Without this root, it'll cause: // pyenv: cannot rehash: $HOME/.pyenv/shims isn't writable `${process.env["HOME"]}/.pyenv`, @@ -17,16 +15,17 @@ function getCommonRoots() { export function execWithSeatbelt( cmd: Array, opts: SpawnOptions, - writableRoots: Array, + writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { let scopedWritePolicy: string; let policyTemplateParams: Array; - if (writableRoots.length > 0) { - // Add `~/.codex` to the list of writable roots - // (if there's any already, not in read-only mode) - getCommonRoots().map((root) => writableRoots.push(root)); - const { policies, params } = writableRoots + + const fullWritableRoots = [...writableRoots, ...getCommonRoots()]; + // In practice, fullWritableRoots will be non-empty, but we check just in + // case the logic to build up fullWritableRoots changes. + if (fullWritableRoots.length > 0) { + const { policies, params } = fullWritableRoots .map((root, index) => ({ policy: `(subpath (param "WRITABLE_ROOT_${index}"))`, param: `-DWRITABLE_ROOT_${index}=${root}`, diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 6cfb304731..e7bcdb2a13 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -21,7 +21,7 @@ const MAX_BUFFER = 1024 * 100; // 100 KB export function exec( command: Array, options: SpawnOptions, - _writableRoots: Array, + _writableRoots: ReadonlyArray, abortSignal?: AbortSignal, ): Promise { // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) From 4025cb6909e2cc90234918dfdfeb07cebf2f6365 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 18:29:11 -0700 Subject: [PATCH 30/84] remove unnecessary isLoggingEnabled() checks --- .../chat/terminal-chat-input-thinking.tsx | 18 ++--- .../components/chat/terminal-chat-input.tsx | 18 ++--- .../chat/terminal-chat-new-input.tsx | 18 ++--- .../src/components/chat/terminal-chat.tsx | 65 +++++++------------ codex-cli/src/utils/agent/agent-loop.ts | 58 +++++++---------- .../src/utils/agent/handle-exec-command.ts | 33 +++++----- codex-cli/src/utils/agent/log.ts | 8 +++ .../src/utils/agent/platform-commands.ts | 10 +-- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 19 ++---- codex-cli/src/utils/config.ts | 14 ++-- 10 files changed, 103 insertions(+), 158 deletions(-) diff --git a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx index 213dd8c9af..fdc8bd218c 100644 --- a/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input-thinking.tsx @@ -1,4 +1,4 @@ -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { Box, Text, useInput, useStdin } from "ink"; import React, { useState } from "react"; import { useInterval } from "use-interval"; @@ -40,11 +40,9 @@ export default function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -65,15 +63,11 @@ export default function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-input.tsx b/codex-cli/src/components/chat/terminal-chat-input.tsx index e1bcbf9ea4..525bbd9c23 100644 --- a/codex-cli/src/components/chat/terminal-chat-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-input.tsx @@ -6,7 +6,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -692,11 +692,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -721,15 +719,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat-new-input.tsx b/codex-cli/src/components/chat/terminal-chat-new-input.tsx index 57acba3625..7dbe130e2c 100644 --- a/codex-cli/src/components/chat/terminal-chat-new-input.tsx +++ b/codex-cli/src/components/chat/terminal-chat-new-input.tsx @@ -8,7 +8,7 @@ import type { import MultilineTextEditor from "./multiline-editor"; import { TerminalChatCommandReview } from "./terminal-chat-command-review.js"; -import { log, isLoggingEnabled } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { loadConfig } from "../../utils/config.js"; import { createInputItem } from "../../utils/input-utils.js"; import { setSessionId } from "../../utils/session.js"; @@ -505,11 +505,9 @@ function TerminalChatInputThinking({ const str = Buffer.isBuffer(data) ? data.toString("utf8") : data; if (str === "\x1b\x1b") { // Treat as the first Escape press – prompt the user for confirmation. - if (isLoggingEnabled()) { - log( - "raw stdin: received collapsed ESC ESC – starting confirmation timer", - ); - } + log( + "raw stdin: received collapsed ESC ESC – starting confirmation timer", + ); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } @@ -531,15 +529,11 @@ function TerminalChatInputThinking({ } if (awaitingConfirm) { - if (isLoggingEnabled()) { - log("useInput: second ESC detected – triggering onInterrupt()"); - } + log("useInput: second ESC detected – triggering onInterrupt()"); onInterrupt(); setAwaitingConfirm(false); } else { - if (isLoggingEnabled()) { - log("useInput: first ESC detected – waiting for confirmation"); - } + log("useInput: first ESC detected – waiting for confirmation"); setAwaitingConfirm(true); setTimeout(() => setAwaitingConfirm(false), 1500); } diff --git a/codex-cli/src/components/chat/terminal-chat.tsx b/codex-cli/src/components/chat/terminal-chat.tsx index fd4cff5df5..825d0b9a92 100644 --- a/codex-cli/src/components/chat/terminal-chat.tsx +++ b/codex-cli/src/components/chat/terminal-chat.tsx @@ -15,7 +15,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import { useConfirmation } from "../../hooks/use-confirmation.js"; import { useTerminalSize } from "../../hooks/use-terminal-size.js"; import { AgentLoop } from "../../utils/agent/agent-loop.js"; -import { isLoggingEnabled, log } from "../../utils/agent/log.js"; +import { log } from "../../utils/agent/log.js"; import { ReviewDecision } from "../../utils/agent/review.js"; import { generateCompactSummary } from "../../utils/compact-summary.js"; import { OPENAI_BASE_URL } from "../../utils/config.js"; @@ -207,30 +207,25 @@ export default function TerminalChat({ // ──────────────────────────────────────────────────────────────── // DEBUG: log every render w/ key bits of state // ──────────────────────────────────────────────────────────────── - if (isLoggingEnabled()) { - log( - `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ - items.length - }`, - ); - } + log( + `render – agent? ${Boolean(agentRef.current)} loading=${loading} items=${ + items.length + }`, + ); useEffect(() => { // Skip recreating the agent if awaiting a decision on a pending confirmation if (confirmationPrompt != null) { - if (isLoggingEnabled()) { - log("skip AgentLoop recreation due to pending confirmationPrompt"); - } + log("skip AgentLoop recreation due to pending confirmationPrompt"); return; } - if (isLoggingEnabled()) { - log("creating NEW AgentLoop"); - log( - `model=${model} instructions=${Boolean( - config.instructions, - )} approvalPolicy=${approvalPolicy}`, - ); - } + + log("creating NEW AgentLoop"); + log( + `model=${model} instructions=${Boolean( + config.instructions, + )} approvalPolicy=${approvalPolicy}`, + ); // Tear down any existing loop before creating a new one agentRef.current?.terminate(); @@ -298,14 +293,10 @@ export default function TerminalChat({ // force a render so JSX below can "see" the freshly created agent forceUpdate(); - if (isLoggingEnabled()) { - log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); - } + log(`AgentLoop created: ${inspect(agentRef.current, { depth: 1 })}`); return () => { - if (isLoggingEnabled()) { - log("terminating AgentLoop"); - } + log("terminating AgentLoop"); agentRef.current?.terminate(); agentRef.current = undefined; forceUpdate(); // re‑render after teardown too @@ -387,9 +378,7 @@ export default function TerminalChat({ // Let's also track whenever the ref becomes available const agent = agentRef.current; useEffect(() => { - if (isLoggingEnabled()) { - log(`agentRef.current is now ${Boolean(agent)}`); - } + log(`agentRef.current is now ${Boolean(agent)}`); }, [agent]); // --------------------------------------------------------------------- @@ -534,11 +523,9 @@ export default function TerminalChat({ if (!agent) { return; } - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); agent.cancel(); setLoading(false); @@ -574,13 +561,11 @@ export default function TerminalChat({ currentModel={model} hasLastResponse={Boolean(lastResponseId)} onSelect={(newModel) => { - if (isLoggingEnabled()) { - log( - "TerminalChat: interruptAgent invoked – calling agent.cancel()", - ); - if (!agent) { - log("TerminalChat: agent is not ready yet"); - } + log( + "TerminalChat: interruptAgent invoked – calling agent.cancel()", + ); + if (!agent) { + log("TerminalChat: agent is not ready yet"); } agent?.cancel(); setLoading(false); diff --git a/codex-cli/src/utils/agent/agent-loop.ts b/codex-cli/src/utils/agent/agent-loop.ts index cbe3c56960..172ba67295 100644 --- a/codex-cli/src/utils/agent/agent-loop.ts +++ b/codex-cli/src/utils/agent/agent-loop.ts @@ -8,7 +8,7 @@ import type { } from "openai/resources/responses/responses.mjs"; import type { Reasoning } from "openai/resources.mjs"; -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; import { OPENAI_BASE_URL, OPENAI_TIMEOUT_MS } from "../config.js"; import { parseToolCallArguments } from "../parsers.js"; import { @@ -116,15 +116,13 @@ export class AgentLoop { // Reset the current stream to allow new requests this.currentStream = null; - if (isLoggingEnabled()) { - log( - `AgentLoop.cancel() invoked – currentStream=${Boolean( - this.currentStream, - )} execAbortController=${Boolean( - this.execAbortController, - )} generation=${this.generation}`, - ); - } + log( + `AgentLoop.cancel() invoked – currentStream=${Boolean( + this.currentStream, + )} execAbortController=${Boolean(this.execAbortController)} generation=${ + this.generation + }`, + ); ( this.currentStream as { controller?: { abort?: () => void } } | null )?.controller?.abort?.(); @@ -136,9 +134,7 @@ export class AgentLoop { // Create a new abort controller for future tool calls this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log("AgentLoop.cancel(): execAbortController.abort() called"); - } + log("AgentLoop.cancel(): execAbortController.abort() called"); // NOTE: We intentionally do *not* clear `lastResponseId` here. If the // stream produced a `function_call` before the user cancelled, OpenAI now @@ -174,9 +170,7 @@ export class AgentLoop { // this.onItem(cancelNotice); this.generation += 1; - if (isLoggingEnabled()) { - log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); - } + log(`AgentLoop.cancel(): generation bumped to ${this.generation}`); } /** @@ -315,13 +309,11 @@ export class AgentLoop { const callId: string = (item as any).call_id ?? (item as any).id; const args = parseToolCallArguments(rawArguments ?? "{}"); - if (isLoggingEnabled()) { - log( - `handleFunctionCall(): name=${ - name ?? "undefined" - } callId=${callId} args=${rawArguments}`, - ); - } + log( + `handleFunctionCall(): name=${ + name ?? "undefined" + } callId=${callId} args=${rawArguments}`, + ); if (args == null) { const outputItem: ResponseInputItem.FunctionCallOutput = { @@ -407,11 +399,9 @@ export class AgentLoop { // Create a fresh AbortController for this run so that tool calls from a // previous run do not accidentally get signalled. this.execAbortController = new AbortController(); - if (isLoggingEnabled()) { - log( - `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, - ); - } + log( + `AgentLoop.run(): new execAbortController created (${this.execAbortController.signal}) for generation ${this.generation}`, + ); // NOTE: We no longer (re‑)attach an `abort` listener to `hardAbort` here. // A single listener that forwards the `abort` to the current // `execAbortController` is installed once in the constructor. Re‑adding a @@ -502,11 +492,9 @@ export class AgentLoop { const mergedInstructions = [prefix, this.instructions] .filter(Boolean) .join("\n"); - if (isLoggingEnabled()) { - log( - `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, - ); - } + log( + `instructions (length ${mergedInstructions.length}): ${mergedInstructions}`, + ); // eslint-disable-next-line no-await-in-loop stream = await this.oai.responses.create({ model: this.model, @@ -733,9 +721,7 @@ export class AgentLoop { try { // eslint-disable-next-line no-await-in-loop for await (const event of stream) { - if (isLoggingEnabled()) { - log(`AgentLoop.run(): response event ${event.type}`); - } + log(`AgentLoop.run(): response event ${event.type}`); // process and surface each item (no‑op until we can depend on streaming events) if (event.type === "response.output_item.done") { diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index 1af390e226..97b78c0817 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -223,23 +223,22 @@ async function execCommand( workdir = process.cwd(); } } - if (isLoggingEnabled()) { - if (applyPatchCommand != null) { - log("EXEC running apply_patch command"); - } else { - const { cmd, timeoutInMillis } = execInput; - // Seconds are a bit easier to read in log messages and most timeouts - // are specified as multiples of 1000, anyway. - const timeout = - timeoutInMillis != null - ? Math.round(timeoutInMillis / 1000).toString() - : "undefined"; - log( - `EXEC running \`${formatCommandForDisplay( - cmd, - )}\` in workdir=${workdir} with timeout=${timeout}s`, - ); - } + + if (applyPatchCommand != null) { + log("EXEC running apply_patch command"); + } else if (isLoggingEnabled()) { + const { cmd, timeoutInMillis } = execInput; + // Seconds are a bit easier to read in log messages and most timeouts + // are specified as multiples of 1000, anyway. + const timeout = + timeoutInMillis != null + ? Math.round(timeoutInMillis / 1000).toString() + : "undefined"; + log( + `EXEC running \`${formatCommandForDisplay( + cmd, + )}\` in workdir=${workdir} with timeout=${timeout}s`, + ); } // Note execApplyPatch() and exec() are coded defensively and should not diff --git a/codex-cli/src/utils/agent/log.ts b/codex-cli/src/utils/agent/log.ts index e804386566..e4aca06c79 100644 --- a/codex-cli/src/utils/agent/log.ts +++ b/codex-cli/src/utils/agent/log.ts @@ -124,6 +124,14 @@ export function log(message: string): void { (logger ?? initLogger()).log(message); } +/** + * USE SPARINGLY! This function should only be used to guard a call to log() if + * the log message is large and you want to avoid constructing it if logging is + * disabled. + * + * `log()` is already a no-op if DEBUG is not set, so an extra + * `isLoggingEnabled()` check is unnecessary. + */ export function isLoggingEnabled(): boolean { return (logger ?? initLogger()).isLoggingEnabled(); } diff --git a/codex-cli/src/utils/agent/platform-commands.ts b/codex-cli/src/utils/agent/platform-commands.ts index 7be02c7ac5..085c575d44 100644 --- a/codex-cli/src/utils/agent/platform-commands.ts +++ b/codex-cli/src/utils/agent/platform-commands.ts @@ -2,7 +2,7 @@ * Utility functions for handling platform-specific commands */ -import { log, isLoggingEnabled } from "./log.js"; +import { log } from "./log.js"; /** * Map of Unix commands to their Windows equivalents @@ -59,9 +59,7 @@ export function adaptCommandForPlatform(command: Array): Array { return command; } - if (isLoggingEnabled()) { - log(`Adapting command '${cmd}' for Windows platform`); - } + log(`Adapting command '${cmd}' for Windows platform`); // Create a new command array with the adapted command const adaptedCommand = [...command]; @@ -78,9 +76,7 @@ export function adaptCommandForPlatform(command: Array): Array { } } - if (isLoggingEnabled()) { - log(`Adapted command: ${adaptedCommand.join(" ")}`); - } + log(`Adapted command: ${adaptedCommand.join(" ")}`); return adaptedCommand; } diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index e7bcdb2a13..35ae8a4a92 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -7,7 +7,7 @@ import type { StdioPipe, } from "child_process"; -import { log, isLoggingEnabled } from "../log.js"; +import { log } from "../log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; @@ -27,10 +27,7 @@ export function exec( // Adapt command for the current platform (e.g., convert 'ls' to 'dir' on Windows) const adaptedCommand = adaptCommandForPlatform(command); - if ( - isLoggingEnabled() && - JSON.stringify(adaptedCommand) !== JSON.stringify(command) - ) { + if (JSON.stringify(adaptedCommand) !== JSON.stringify(command)) { log( `Command adapted for platform: ${command.join( " ", @@ -95,9 +92,7 @@ export function exec( // timely fashion. if (abortSignal) { const abortHandler = () => { - if (isLoggingEnabled()) { - log(`raw-exec: abort signal received – killing child ${child.pid}`); - } + log(`raw-exec: abort signal received – killing child ${child.pid}`); const killTarget = (signal: NodeJS.Signals) => { if (!child.pid) { return; @@ -194,11 +189,9 @@ export function exec( exitCode = 1; } - if (isLoggingEnabled()) { - log( - `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, - ); - } + log( + `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, + ); resolve({ stdout, stderr, diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index 8467653580..3be77059e0 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -8,7 +8,7 @@ import type { FullAutoErrorMode } from "./auto-approval-mode.js"; -import { log, isLoggingEnabled } from "./agent/log.js"; +import { log } from "./agent/log.js"; import { AutoApprovalMode } from "./auto-approval-mode.js"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { load as loadYaml, dump as dumpYaml } from "js-yaml"; @@ -245,15 +245,11 @@ export const loadConfig = ( ? resolvePath(cwd, options.projectDocPath) : discoverProjectDocPath(cwd); if (projectDocPath) { - if (isLoggingEnabled()) { - log( - `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, - ); - } + log( + `[codex] Loaded project doc from ${projectDocPath} (${projectDoc.length} bytes)`, + ); } else { - if (isLoggingEnabled()) { - log(`[codex] No project doc found in ${cwd}`); - } + log(`[codex] No project doc found in ${cwd}`); } } From 698381bae42bc5c93167af819de46caac9aab911 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 19 Apr 2025 18:29:11 -0700 Subject: [PATCH 31/84] reduce max output of ExecResult --- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 130 +++++++++++++----- 1 file changed, 96 insertions(+), 34 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index 35ae8a4a92..22add83ed8 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -12,7 +12,10 @@ import { adaptCommandForPlatform } from "../platform-commands.js"; import { spawn } from "child_process"; import * as os from "os"; -const MAX_BUFFER = 1024 * 100; // 100 KB +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; /** * This function should never return a rejected promise: errors should be @@ -143,37 +146,14 @@ export function exec( // resolve the promise and translate the failure into a regular // ExecResult object so the rest of the agent loop can carry on gracefully. - const stdoutChunks: Array = []; - const stderrChunks: Array = []; - let numStdoutBytes = 0; - let numStderrBytes = 0; - let hitMaxStdout = false; - let hitMaxStderr = false; - return new Promise((resolve) => { - child.stdout?.on("data", (data: Buffer) => { - if (!hitMaxStdout) { - numStdoutBytes += data.length; - if (numStdoutBytes <= MAX_BUFFER) { - stdoutChunks.push(data); - } else { - hitMaxStdout = true; - } - } - }); - child.stderr?.on("data", (data: Buffer) => { - if (!hitMaxStderr) { - numStderrBytes += data.length; - if (numStderrBytes <= MAX_BUFFER) { - stderrChunks.push(data); - } else { - hitMaxStderr = true; - } - } - }); + // Collect stdout and stderr up to configured limits. + const stdoutCollector = createTruncatingCollector(child.stdout!); + const stderrCollector = createTruncatingCollector(child.stderr!); + child.on("exit", (code, signal) => { - const stdout = Buffer.concat(stdoutChunks).toString("utf8"); - const stderr = Buffer.concat(stderrChunks).toString("utf8"); + const stdout = stdoutCollector.getString(); + const stderr = stderrCollector.getString(); // Map (code, signal) to an exit code. We expect exactly one of the two // values to be non-null, but we code defensively to handle the case where @@ -192,19 +172,101 @@ export function exec( log( `raw-exec: child ${child.pid} exited code=${exitCode} signal=${signal}`, ); - resolve({ + + const execResult = { stdout, stderr, exitCode, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); child.on("error", (err) => { - resolve({ + const execResult = { stdout: "", stderr: String(err), exitCode: 1, - }); + }; + resolve( + addTruncationWarningsIfNecessary( + execResult, + stdoutCollector.hit, + stderrCollector.hit, + ), + ); }); }); } + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +function createTruncatingCollector( + stream: NodeJS.ReadableStream, + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +) { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + totalBytes += data.length; + for (let i = 0; i < data.length; i++) { + if (data[i] === 0x0a) { + totalLines++; + } + } + if (totalBytes <= byteLimit && totalLines <= lineLimit) { + chunks.push(data); + } else { + hitLimit = true; + } + }); + + return { + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} + +/** + * Adds a truncation warnings to stdout and stderr, if appropriate. + */ +function addTruncationWarningsIfNecessary( + execResult: ExecResult, + hitMaxStdout: boolean, + hitMaxStderr: boolean, +): ExecResult { + if (!hitMaxStdout && !hitMaxStderr) { + return execResult; + } else { + const { stdout, stderr, exitCode } = execResult; + return { + stdout: hitMaxStdout + ? stdout + "\n\n[Output truncated: too many lines or bytes]" + : stdout, + stderr: hitMaxStderr + ? stderr + "\n\n[Output truncated: too many lines or bytes]" + : stderr, + exitCode, + }; + } +} From b54367d34b192ebe26943a53ab3f1124cde2c44a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 09:47:58 -0700 Subject: [PATCH 32/84] revert #386 due to unsafe shell command parsing --- README.md | 3 --- codex-cli/src/approvals.ts | 19 ------------------- codex-cli/src/utils/config.ts | 16 ---------------- codex-cli/tests/approvals.test.ts | 31 +------------------------------ 4 files changed, 1 insertion(+), 68 deletions(-) diff --git a/README.md b/README.md index 2094b9398c..e5690d7d48 100644 --- a/README.md +++ b/README.md @@ -289,9 +289,6 @@ model: o4-mini # Default model approvalMode: suggest # or auto-edit, full-auto fullAutoErrorMode: ask-user # or ignore-and-continue notify: true # Enable desktop notifications for responses -safeCommands: - - npm test # Automatically approve npm test - - yarn lint # Automatically approve yarn lint ``` ```json diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index b9eb50d1d7..f4a35402f9 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -4,7 +4,6 @@ import { identify_files_added, identify_files_needed, } from "./utils/agent/apply-patch"; -import { loadConfig } from "./utils/config"; import * as path from "path"; import { parse } from "shell-quote"; @@ -297,24 +296,6 @@ export function isSafeCommand( ): SafeCommandReason | null { const [cmd0, cmd1, cmd2, cmd3] = command; - const config = loadConfig(); - if (config.safeCommands && Array.isArray(config.safeCommands)) { - for (const safe of config.safeCommands) { - // safe: "npm test" → ["npm", "test"] - const safeArr = typeof safe === "string" ? safe.trim().split(/\s+/) : []; - if ( - safeArr.length > 0 && - safeArr.length <= command.length && - safeArr.every((v, i) => v === command[i]) - ) { - return { - reason: "User-defined safe command", - group: "User config", - }; - } - } - } - switch (cmd0) { case "cd": return { diff --git a/codex-cli/src/utils/config.ts b/codex-cli/src/utils/config.ts index a4a9c0cb0e..190d111762 100644 --- a/codex-cli/src/utils/config.ts +++ b/codex-cli/src/utils/config.ts @@ -78,8 +78,6 @@ export type StoredConfig = { saveHistory?: boolean; sensitivePatterns?: Array; }; - /** User-defined safe commands */ - safeCommands?: Array; }; // Minimal config written on first run. An *empty* model string ensures that @@ -113,8 +111,6 @@ export type AppConfig = { saveHistory: boolean; sensitivePatterns: Array; }; - /** User-defined safe commands */ - safeCommands?: Array; }; // --------------------------------------------------------------------------- @@ -297,7 +293,6 @@ export const loadConfig = ( instructions: combinedInstructions, notify: storedConfig.notify === true, approvalMode: storedConfig.approvalMode, - safeCommands: storedConfig.safeCommands ?? [], }; // ----------------------------------------------------------------------- @@ -375,13 +370,6 @@ export const loadConfig = ( }; } - // Load user-defined safe commands - if (Array.isArray(storedConfig.safeCommands)) { - config.safeCommands = storedConfig.safeCommands.map(String); - } else { - config.safeCommands = []; - } - return config; }; @@ -425,10 +413,6 @@ export const saveConfig = ( sensitivePatterns: config.history.sensitivePatterns, }; } - // Save: User-defined safe commands - if (config.safeCommands && config.safeCommands.length > 0) { - configToSave.safeCommands = config.safeCommands; - } if (ext === ".yaml" || ext === ".yml") { writeFileSync(targetPath, dumpYaml(configToSave), "utf-8"); diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index 43490cf84e..a39adff460 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -1,13 +1,7 @@ import type { SafetyAssessment } from "../src/approvals"; import { canAutoApprove } from "../src/approvals"; -import { describe, test, expect, vi } from "vitest"; - -vi.mock("../src/utils/config", () => ({ - loadConfig: () => ({ - safeCommands: ["npm test", "sl"], - }), -})); +import { describe, test, expect } from "vitest"; describe("canAutoApprove()", () => { const env = { @@ -95,27 +89,4 @@ describe("canAutoApprove()", () => { expect(check(["cargo", "build"])).toEqual({ type: "ask-user" }); }); - - test("commands in safeCommands config should be safe", async () => { - expect(check(["npm", "test"])).toEqual({ - type: "auto-approve", - reason: "User-defined safe command", - group: "User config", - runInSandbox: false, - }); - - expect(check(["sl"])).toEqual({ - type: "auto-approve", - reason: "User-defined safe command", - group: "User config", - runInSandbox: false, - }); - - expect(check(["npm", "test", "--watch"])).toEqual({ - type: "auto-approve", - reason: "User-defined safe command", - group: "User config", - runInSandbox: false, - }); - }); }); From cb1e6383064b9a2531a19f70f68ae734d207d4f6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 10:15:48 -0700 Subject: [PATCH 33/84] do not auto-approve the find command if it contains options that write files or spawn commands --- codex-cli/src/approvals.ts | 34 +++++++++++++++++--- codex-cli/tests/approvals.test.ts | 52 +++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index f4a35402f9..ff37a8903f 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -329,11 +329,20 @@ export function isSafeCommand( reason: "Ripgrep search", group: "Searching", }; - case "find": - return { - reason: "Find files or directories", - group: "Searching", - }; + case "find": { + // Certain options to `find` allow executing arbitrary processes, so we + // cannot auto-approve them. + if ( + command.some((arg: string) => UNSAFE_OPTIONS_FOR_FIND_COMMAND.has(arg)) + ) { + break; + } else { + return { + reason: "Find files or directories", + group: "Searching", + }; + } + } case "grep": return { reason: "Text search (grep)", @@ -421,6 +430,21 @@ function isValidSedNArg(arg: string | undefined): boolean { return arg != null && /^(\d+,)?\d+p$/.test(arg); } +const UNSAFE_OPTIONS_FOR_FIND_COMMAND: ReadonlySet = new Set([ + // Options that can execute arbitrary commands. + "-exec", + "-execdir", + "-ok", + "-okdir", + // Option that deletes matching files. + "-delete", + // Options that write pathnames to a file. + "-fls", + "-fprint", + "-fprint0", + "-fprintf", +]); + // ---------------- Helper utilities for complex shell expressions ----------------- // A conservative allow-list of bash operators that do not, on their own, cause diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index a39adff460..a90abad6eb 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -89,4 +89,56 @@ describe("canAutoApprove()", () => { expect(check(["cargo", "build"])).toEqual({ type: "ask-user" }); }); + + test("find", () => { + expect(check(["find", ".", "-name", "file.txt"])).toEqual({ + type: "auto-approve", + reason: "Find files or directories", + group: "Searching", + runInSandbox: false, + }); + + // Options that can execute arbitrary commands. + expect( + check(["find", ".", "-name", "file.txt", "-exec", "rm", "{}", ";"]), + ).toEqual({ + type: "ask-user", + }); + expect( + check(["find", ".", "-name", "*.py", "-execdir", "python3", "{}", ";"]), + ).toEqual({ + type: "ask-user", + }); + expect( + check(["find", ".", "-name", "file.txt", "-ok", "rm", "{}", ";"]), + ).toEqual({ + type: "ask-user", + }); + expect( + check(["find", ".", "-name", "*.py", "-okdir", "python3", "{}", ";"]), + ).toEqual({ + type: "ask-user", + }); + + // Option that deletes matching files. + expect(check(["find", ".", "-delete", "-name", "file.txt"])).toEqual({ + type: "ask-user", + }); + + // Options that write pathnames to a file. + expect(check(["find", ".", "-fls", "/etc/passwd"])).toEqual({ + type: "ask-user", + }); + expect(check(["find", ".", "-fprint", "/etc/passwd"])).toEqual({ + type: "ask-user", + }); + expect(check(["find", ".", "-fprint0", "/etc/passwd"])).toEqual({ + type: "ask-user", + }); + expect( + check(["find", ".", "-fprintf", "/root/suid.txt", "%#m %u %p\n"]), + ).toEqual({ + type: "ask-user", + }); + }); }); From a1afcb6916fa5b17d12d20637718b03ffd968969 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 16:37:23 -0700 Subject: [PATCH 34/84] add instructions for connecting to a visual debugger under Contributing --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 1abae778cf..e07db1b02f 100644 --- a/README.md +++ b/README.md @@ -448,6 +448,15 @@ pnpm lint:fix pnpm format:fix ``` +### Debugging + +To debug the CLI with a visual debugger, do the following in the `codex-cli` folder: + +- Build: run `pnpm run build`, which will generate `cli.js.map` alongside `cli.js` +- Run the CLI with `node --inspect-brk ./dist/cli.js` The program then waits until a debugger is attached before proceeding. Options: + - In VS Code, choose **Debug: Attach to Node Process** from the command palette and choose the option in the dropdown with debug port `9229` (likely the first option) + - Go to in Chrome and find **localhost:9229** and click **trace** + #### Nix Flake Development Prerequisite: Nix >= 2.4 with flakes enabled (`experimental-features = nix-command flakes` in `~/.config/nix/nix.conf`). From d13a3708b3634c12449d7406a82a52482558e2ef Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 16:58:28 -0700 Subject: [PATCH 35/84] include fractional portion of chunk that exceeds stdout/stderr limit --- codex-cli/src/utils/agent/sandbox/raw-exec.ts | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index b3d1d8ec25..a04cd0b4e1 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -223,15 +223,42 @@ function createTruncatingCollector( if (hitLimit) { return; } - totalBytes += data.length; - for (let i = 0; i < data.length; i++) { + const dataLength = data.length; + let newlineCount = 0; + for (let i = 0; i < dataLength; i++) { if (data[i] === 0x0a) { - totalLines++; + newlineCount++; } } - if (totalBytes <= byteLimit && totalLines <= lineLimit) { + // If entire chunk fits within byte and line limits, take it whole + if (totalBytes + dataLength <= byteLimit && totalLines + newlineCount <= lineLimit) { chunks.push(data); + totalBytes += dataLength; + totalLines += newlineCount; } else { + // Otherwise, take a partial slice up to the first limit breach + const allowedBytes = byteLimit - totalBytes; + const allowedLines = lineLimit - totalLines; + let bytesTaken = 0; + let linesSeen = 0; + for (let i = 0; i < dataLength; i++) { + if (bytesTaken === allowedBytes) { + break; + } + const byte = data[i]; + if (byte === 0x0a) { + if (linesSeen === allowedLines) { + break; + } + linesSeen++; + } + bytesTaken++; + } + if (bytesTaken > 0) { + chunks.push(data.slice(0, bytesTaken)); + totalBytes += bytesTaken; + totalLines += linesSeen; + } hitLimit = true; } }); From 0f9dfe3b8ae64e2274411d67be976eed028d9821 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 16:58:28 -0700 Subject: [PATCH 36/84] include fractional portion of chunk that exceeds stdout/stderr limit --- .../sandbox/create-truncating-collector.ts | 72 +++++++++++++++++++ codex-cli/src/utils/agent/sandbox/raw-exec.ts | 44 +----------- .../tests/create-truncating-collector.test.ts | 55 ++++++++++++++ 3 files changed, 128 insertions(+), 43 deletions(-) create mode 100644 codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts create mode 100644 codex-cli/tests/create-truncating-collector.test.ts diff --git a/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts new file mode 100644 index 0000000000..2e8b6b63e7 --- /dev/null +++ b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts @@ -0,0 +1,72 @@ +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +export function createTruncatingCollector( + stream: NodeJS.ReadableStream, + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +) { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + const dataLength = data.length; + let newlineCount = 0; + for (let i = 0; i < dataLength; i++) { + if (data[i] === 0x0a) { + newlineCount++; + } + } + // If entire chunk fits within byte and line limits, take it whole + if ( + totalBytes + dataLength <= byteLimit && + totalLines + newlineCount <= lineLimit + ) { + chunks.push(data); + totalBytes += dataLength; + totalLines += newlineCount; + } else { + // Otherwise, take a partial slice up to the first limit breach + const allowedBytes = byteLimit - totalBytes; + const allowedLines = lineLimit - totalLines; + let bytesTaken = 0; + let linesSeen = 0; + for (let i = 0; i < dataLength; i++) { + if (bytesTaken === allowedBytes) { + break; + } + const byte = data[i]; + if (byte === 0x0a) { + if (linesSeen === allowedLines) { + break; + } + linesSeen++; + } + bytesTaken++; + } + if (bytesTaken > 0) { + chunks.push(data.slice(0, bytesTaken)); + totalBytes += bytesTaken; + totalLines += linesSeen; + } + hitLimit = true; + } + }); + + return { + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index b3d1d8ec25..9917536f65 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -9,6 +9,7 @@ import type { import { log } from "../../logger/log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; +import { createTruncatingCollector } from "./create-truncating-collector"; import { spawn } from "child_process"; import * as os from "os"; @@ -204,49 +205,6 @@ export function exec( }); } -/** - * Creates a collector that accumulates data Buffers from a stream up to - * specified byte and line limits. After either limit is exceeded, further - * data is ignored. - */ -function createTruncatingCollector( - stream: NodeJS.ReadableStream, - byteLimit: number = MAX_OUTPUT_BYTES, - lineLimit: number = MAX_OUTPUT_LINES, -) { - const chunks: Array = []; - let totalBytes = 0; - let totalLines = 0; - let hitLimit = false; - - stream?.on("data", (data: Buffer) => { - if (hitLimit) { - return; - } - totalBytes += data.length; - for (let i = 0; i < data.length; i++) { - if (data[i] === 0x0a) { - totalLines++; - } - } - if (totalBytes <= byteLimit && totalLines <= lineLimit) { - chunks.push(data); - } else { - hitLimit = true; - } - }); - - return { - getString() { - return Buffer.concat(chunks).toString("utf8"); - }, - /** True if either byte or line limit was exceeded */ - get hit(): boolean { - return hitLimit; - }, - }; -} - /** * Adds a truncation warnings to stdout and stderr, if appropriate. */ diff --git a/codex-cli/tests/create-truncating-collector.test.ts b/codex-cli/tests/create-truncating-collector.test.ts new file mode 100644 index 0000000000..ad3dee558f --- /dev/null +++ b/codex-cli/tests/create-truncating-collector.test.ts @@ -0,0 +1,55 @@ +import { PassThrough } from "stream"; +import { once } from "events"; +import { describe, it, expect } from "vitest"; +import { createTruncatingCollector } from "../src/utils/agent/sandbox/create-truncating-collector.js"; + +describe("createTruncatingCollector", () => { + it("collects data under limits without truncation", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 100, 10); + const data = "line1\nline2\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe(data); + expect(collector.hit).toBe(false); + }); + + it("truncates data over byte limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 5, 100); + stream.end(Buffer.from("hello world")); + await once(stream, "end"); + expect(collector.getString()).toBe("hello"); + expect(collector.hit).toBe(true); + }); + + it("truncates data over line limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 1000, 2); + const data = "a\nb\nc\nd\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe("a\nb\n"); + expect(collector.hit).toBe(true); + }); + + it("stops collecting after limit is hit across multiple writes", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 10, 2); + stream.write(Buffer.from("1\n")); + stream.write(Buffer.from("2\n3\n4\n")); + stream.end(); + await once(stream, "end"); + expect(collector.getString()).toBe("1\n2\n"); + expect(collector.hit).toBe(true); + }); + + it("handles zero limits", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 0, 0); + stream.end(Buffer.from("anything\n")); + await once(stream, "end"); + expect(collector.getString()).toBe(""); + expect(collector.hit).toBe(true); + }); +}); From 5fa24ca8ab1d1ca418c4e06665ada795ffb4581a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 16:58:28 -0700 Subject: [PATCH 37/84] include fractional portion of chunk that exceeds stdout/stderr limit --- .../sandbox/create-truncating-collector.ts | 80 +++++++++++++++++++ codex-cli/src/utils/agent/sandbox/raw-exec.ts | 49 +----------- .../tests/create-truncating-collector.test.ts | 55 +++++++++++++ 3 files changed, 136 insertions(+), 48 deletions(-) create mode 100644 codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts create mode 100644 codex-cli/tests/create-truncating-collector.test.ts diff --git a/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts new file mode 100644 index 0000000000..ed98444b91 --- /dev/null +++ b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts @@ -0,0 +1,80 @@ +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +export function createTruncatingCollector( + stream: NodeJS.ReadableStream, + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +): { + getString: () => string; + hit: boolean; +} { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + const dataLength = data.length; + let newlineCount = 0; + for (let i = 0; i < dataLength; i++) { + if (data[i] === 0x0a) { + newlineCount++; + } + } + // If entire chunk fits within byte and line limits, take it whole + if ( + totalBytes + dataLength <= byteLimit && + totalLines + newlineCount <= lineLimit + ) { + chunks.push(data); + totalBytes += dataLength; + totalLines += newlineCount; + } else { + // Otherwise, take a partial slice up to the first limit breach + const allowedBytes = byteLimit - totalBytes; + const allowedLines = lineLimit - totalLines; + let bytesTaken = 0; + let linesSeen = 0; + for (let i = 0; i < dataLength; i++) { + if (bytesTaken === allowedBytes) { + break; + } + const byte = data[i]; + if (byte === 0x0a) { + if (linesSeen === allowedLines) { + break; + } + linesSeen++; + } + bytesTaken++; + } + if (bytesTaken > 0) { + chunks.push(data.slice(0, bytesTaken)); + totalBytes += bytesTaken; + totalLines += linesSeen; + } + hitLimit = true; + } + }); + + return { + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index b3d1d8ec25..b33feb8518 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -9,14 +9,10 @@ import type { import { log } from "../../logger/log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; +import { createTruncatingCollector } from "./create-truncating-collector"; import { spawn } from "child_process"; import * as os from "os"; -// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, -// whichever limit is reached first. -const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB -const MAX_OUTPUT_LINES = 256; - /** * This function should never return a rejected promise: errors should be * mapped to a non-zero exit code and the error message should be in stderr. @@ -204,49 +200,6 @@ export function exec( }); } -/** - * Creates a collector that accumulates data Buffers from a stream up to - * specified byte and line limits. After either limit is exceeded, further - * data is ignored. - */ -function createTruncatingCollector( - stream: NodeJS.ReadableStream, - byteLimit: number = MAX_OUTPUT_BYTES, - lineLimit: number = MAX_OUTPUT_LINES, -) { - const chunks: Array = []; - let totalBytes = 0; - let totalLines = 0; - let hitLimit = false; - - stream?.on("data", (data: Buffer) => { - if (hitLimit) { - return; - } - totalBytes += data.length; - for (let i = 0; i < data.length; i++) { - if (data[i] === 0x0a) { - totalLines++; - } - } - if (totalBytes <= byteLimit && totalLines <= lineLimit) { - chunks.push(data); - } else { - hitLimit = true; - } - }); - - return { - getString() { - return Buffer.concat(chunks).toString("utf8"); - }, - /** True if either byte or line limit was exceeded */ - get hit(): boolean { - return hitLimit; - }, - }; -} - /** * Adds a truncation warnings to stdout and stderr, if appropriate. */ diff --git a/codex-cli/tests/create-truncating-collector.test.ts b/codex-cli/tests/create-truncating-collector.test.ts new file mode 100644 index 0000000000..ad3dee558f --- /dev/null +++ b/codex-cli/tests/create-truncating-collector.test.ts @@ -0,0 +1,55 @@ +import { PassThrough } from "stream"; +import { once } from "events"; +import { describe, it, expect } from "vitest"; +import { createTruncatingCollector } from "../src/utils/agent/sandbox/create-truncating-collector.js"; + +describe("createTruncatingCollector", () => { + it("collects data under limits without truncation", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 100, 10); + const data = "line1\nline2\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe(data); + expect(collector.hit).toBe(false); + }); + + it("truncates data over byte limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 5, 100); + stream.end(Buffer.from("hello world")); + await once(stream, "end"); + expect(collector.getString()).toBe("hello"); + expect(collector.hit).toBe(true); + }); + + it("truncates data over line limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 1000, 2); + const data = "a\nb\nc\nd\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe("a\nb\n"); + expect(collector.hit).toBe(true); + }); + + it("stops collecting after limit is hit across multiple writes", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 10, 2); + stream.write(Buffer.from("1\n")); + stream.write(Buffer.from("2\n3\n4\n")); + stream.end(); + await once(stream, "end"); + expect(collector.getString()).toBe("1\n2\n"); + expect(collector.hit).toBe(true); + }); + + it("handles zero limits", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 0, 0); + stream.end(Buffer.from("anything\n")); + await once(stream, "end"); + expect(collector.getString()).toBe(""); + expect(collector.hit).toBe(true); + }); +}); From 8ff34daa4891e3597e6f7634dce5f4accaf3c8f2 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 16:58:28 -0700 Subject: [PATCH 38/84] include fractional portion of chunk that exceeds stdout/stderr limit --- .../sandbox/create-truncating-collector.ts | 78 +++++++++++++++++++ codex-cli/src/utils/agent/sandbox/raw-exec.ts | 49 +----------- .../tests/create-truncating-collector.test.ts | 55 +++++++++++++ 3 files changed, 134 insertions(+), 48 deletions(-) create mode 100644 codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts create mode 100644 codex-cli/tests/create-truncating-collector.test.ts diff --git a/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts new file mode 100644 index 0000000000..518d475c78 --- /dev/null +++ b/codex-cli/src/utils/agent/sandbox/create-truncating-collector.ts @@ -0,0 +1,78 @@ +// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, +// whichever limit is reached first. +const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB +const MAX_OUTPUT_LINES = 256; + +/** + * Creates a collector that accumulates data Buffers from a stream up to + * specified byte and line limits. After either limit is exceeded, further + * data is ignored. + */ +export function createTruncatingCollector( + stream: NodeJS.ReadableStream, + byteLimit: number = MAX_OUTPUT_BYTES, + lineLimit: number = MAX_OUTPUT_LINES, +): { + getString: () => string; + hit: boolean; +} { + const chunks: Array = []; + let totalBytes = 0; + let totalLines = 0; + let hitLimit = false; + + stream?.on("data", (data: Buffer) => { + if (hitLimit) { + return; + } + const dataLength = data.length; + let newlineCount = 0; + for (let i = 0; i < dataLength; i++) { + if (data[i] === 0x0a) { + newlineCount++; + } + } + // If entire chunk fits within byte and line limits, take it whole + if ( + totalBytes + dataLength <= byteLimit && + totalLines + newlineCount <= lineLimit + ) { + chunks.push(data); + totalBytes += dataLength; + totalLines += newlineCount; + } else { + // Otherwise, take a partial slice up to the first limit breach + const allowedBytes = byteLimit - totalBytes; + const allowedLines = lineLimit - totalLines; + let bytesTaken = 0; + let linesSeen = 0; + for (let i = 0; i < dataLength; i++) { + // Stop if byte or line limit is reached + if (bytesTaken === allowedBytes || linesSeen === allowedLines) { + break; + } + const byte = data[i]; + if (byte === 0x0a) { + linesSeen++; + } + bytesTaken++; + } + if (bytesTaken > 0) { + chunks.push(data.slice(0, bytesTaken)); + totalBytes += bytesTaken; + totalLines += linesSeen; + } + hitLimit = true; + } + }); + + return { + getString() { + return Buffer.concat(chunks).toString("utf8"); + }, + /** True if either byte or line limit was exceeded */ + get hit(): boolean { + return hitLimit; + }, + }; +} diff --git a/codex-cli/src/utils/agent/sandbox/raw-exec.ts b/codex-cli/src/utils/agent/sandbox/raw-exec.ts index b3d1d8ec25..b33feb8518 100644 --- a/codex-cli/src/utils/agent/sandbox/raw-exec.ts +++ b/codex-cli/src/utils/agent/sandbox/raw-exec.ts @@ -9,14 +9,10 @@ import type { import { log } from "../../logger/log.js"; import { adaptCommandForPlatform } from "../platform-commands.js"; +import { createTruncatingCollector } from "./create-truncating-collector"; import { spawn } from "child_process"; import * as os from "os"; -// Maximum output cap: either MAX_OUTPUT_LINES lines or MAX_OUTPUT_BYTES bytes, -// whichever limit is reached first. -const MAX_OUTPUT_BYTES = 1024 * 10; // 10 KB -const MAX_OUTPUT_LINES = 256; - /** * This function should never return a rejected promise: errors should be * mapped to a non-zero exit code and the error message should be in stderr. @@ -204,49 +200,6 @@ export function exec( }); } -/** - * Creates a collector that accumulates data Buffers from a stream up to - * specified byte and line limits. After either limit is exceeded, further - * data is ignored. - */ -function createTruncatingCollector( - stream: NodeJS.ReadableStream, - byteLimit: number = MAX_OUTPUT_BYTES, - lineLimit: number = MAX_OUTPUT_LINES, -) { - const chunks: Array = []; - let totalBytes = 0; - let totalLines = 0; - let hitLimit = false; - - stream?.on("data", (data: Buffer) => { - if (hitLimit) { - return; - } - totalBytes += data.length; - for (let i = 0; i < data.length; i++) { - if (data[i] === 0x0a) { - totalLines++; - } - } - if (totalBytes <= byteLimit && totalLines <= lineLimit) { - chunks.push(data); - } else { - hitLimit = true; - } - }); - - return { - getString() { - return Buffer.concat(chunks).toString("utf8"); - }, - /** True if either byte or line limit was exceeded */ - get hit(): boolean { - return hitLimit; - }, - }; -} - /** * Adds a truncation warnings to stdout and stderr, if appropriate. */ diff --git a/codex-cli/tests/create-truncating-collector.test.ts b/codex-cli/tests/create-truncating-collector.test.ts new file mode 100644 index 0000000000..ad3dee558f --- /dev/null +++ b/codex-cli/tests/create-truncating-collector.test.ts @@ -0,0 +1,55 @@ +import { PassThrough } from "stream"; +import { once } from "events"; +import { describe, it, expect } from "vitest"; +import { createTruncatingCollector } from "../src/utils/agent/sandbox/create-truncating-collector.js"; + +describe("createTruncatingCollector", () => { + it("collects data under limits without truncation", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 100, 10); + const data = "line1\nline2\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe(data); + expect(collector.hit).toBe(false); + }); + + it("truncates data over byte limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 5, 100); + stream.end(Buffer.from("hello world")); + await once(stream, "end"); + expect(collector.getString()).toBe("hello"); + expect(collector.hit).toBe(true); + }); + + it("truncates data over line limit", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 1000, 2); + const data = "a\nb\nc\nd\n"; + stream.end(Buffer.from(data)); + await once(stream, "end"); + expect(collector.getString()).toBe("a\nb\n"); + expect(collector.hit).toBe(true); + }); + + it("stops collecting after limit is hit across multiple writes", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 10, 2); + stream.write(Buffer.from("1\n")); + stream.write(Buffer.from("2\n3\n4\n")); + stream.end(); + await once(stream, "end"); + expect(collector.getString()).toBe("1\n2\n"); + expect(collector.hit).toBe(true); + }); + + it("handles zero limits", async () => { + const stream = new PassThrough(); + const collector = createTruncatingCollector(stream, 0, 0); + stream.end(Buffer.from("anything\n")); + await once(stream, "end"); + expect(collector.getString()).toBe(""); + expect(collector.hit).toBe(true); + }); +}); From c616733e559f19d54b7be11edeab34da7d6b6bbe Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 23:17:11 -0700 Subject: [PATCH 39/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 157 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 +++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 79 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..d9a7665aae 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#whycodex) +- [Security Model \& Permissions](#securitymodelpermissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#systemrequirements) +- [CLI Reference](#clireference) +- [Memory \& Project Docs](#memoryprojectdocs) +- [Non-interactive / CI mode](#noninteractivecimode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-highimpact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security \& Responsible AI](#securityresponsibleai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
(default) | • Read any file in the repo | • **All** file writes/patches
• **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
• Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
(default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,13 +512,13 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. @@ -527,7 +527,6 @@ No special Git commands, email attachments, or commit footers required. | Scenario | Command | | ----------------- | ----------------------------------------------------------------------------------------- | | Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 835501ad652ceb85a25fe7126ebb3c5c7810207c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 23:17:11 -0700 Subject: [PATCH 40/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..83d5d2e16a 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#whycodex) +- [Security Model \& Permissions](#securitymodelpermissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#systemrequirements) +- [CLI Reference](#clireference) +- [Memory \& Project Docs](#memoryprojectdocs) +- [Non-interactive / CI mode](#noninteractivecimode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-highimpact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security \& Responsible AI](#securityresponsibleai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 66d4d73e24703b3efc0bed7d76b4363ed1b51190 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 23:34:05 -0700 Subject: [PATCH 41/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..00ccead8cf 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#whycodex) +- [Security Model \& Permissions](#securitymodelpermissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#systemrequirements) +- [CLI Reference](#clireference) +- [Memory \& Project Docs](#memoryprojectdocs) +- [Non-interactive / CI mode](#noninteractivecimode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-highimpact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security \& Responsible AI](#securityresponsibleai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 7077f164824f2a97db7de33fc0670987b6928fc3 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 23:34:05 -0700 Subject: [PATCH 42/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..843db3ed50 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model \& Permissions](#security-model-permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory \& Project Docs](#memory-project-docs) +- [Non-interactive / CI mode](#noninteractive-ci-mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-highimpact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security \& Responsible AI](#security-responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From b5dc6314e12931085d490d4296f15d462f9c180c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 21 Apr 2025 23:34:05 -0700 Subject: [PATCH 43/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..9313780aa5 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model & Permissions](#security-model-permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory & Project Docs](#memory-project-docs) +- [Non-interactive / CI mode](#non-interactive-ci-mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security & Responsible AI](#security-responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From d1eba37951ba0ac56d6ed9d6593fc892d9725a19 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 00:09:28 -0700 Subject: [PATCH 44/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..2bb19a1a07 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model & Permissions](#security-model-permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory & Project Docs](#memory-project-docs) +- [Non-interactive / CI mode](#non-interactive-ci-mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security & Responsible AI](#security-responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 0d56667bf90f57f82ef1f0d33e8fb3141991b83f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 00:09:28 -0700 Subject: [PATCH 45/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..c0705f9f06 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model & Permissions](#security--model-permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory & Project Docs](#memory-project-docs) +- [Non-interactive / CI mode](#non-interactive-ci-mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security & Responsible AI](#security--responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 3f1e43cdecc156a4de9f87bc32dcec6c1b17c32c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 00:09:28 -0700 Subject: [PATCH 46/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..3b82ece3b5 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model & Permissions](#security--model--permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory & Project Docs](#memory--project-docs) +- [Non-interactive / CI mode](#non-interactive-ci--mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security & Responsible AI](#security--responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 0645126f303ae2ad4c2fd3c20771d0846c0c651f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 00:09:28 -0700 Subject: [PATCH 47/84] Enforce ASCII in README.md --- .github/workflows/ci.yml | 3 + README.md | 163 +++++++++++++++++++-------------------- scripts/asciicheck.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 82 deletions(-) create mode 100755 scripts/asciicheck.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99c2daa04..1f5dd0d31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,6 @@ jobs: - name: Build run: pnpm run build + + - name: Ensure README.md contains only ASCII and certain Unicode code points + run: ./scripts/asciicheck.py README.md diff --git a/README.md b/README.md index c9fcc93c80..9ac89499ac 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) -- [Why Codex?](#whycodex) -- [Security Model \& Permissions](#securitymodelpermissions) +- [Why Codex?](#why-codex) +- [Security Model & Permissions](#security-model--permissions) - [Platform sandboxing details](#platform-sandboxing-details) -- [System Requirements](#systemrequirements) -- [CLI Reference](#clireference) -- [Memory \& Project Docs](#memoryprojectdocs) -- [Non‑interactive / CI mode](#noninteractivecimode) +- [System Requirements](#system-requirements) +- [CLI Reference](#cli-reference) +- [Memory & Project Docs](#memory--project-docs) +- [Non-interactive / CI mode](#non-interactive--ci-mode) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) @@ -27,7 +27,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Nix Flake Development](#nix-flake-development) - - [Writing high‑impact code changes](#writing-highimpact-code-changes) + - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) - [Community values](#community-values) @@ -35,7 +35,7 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) -- [Security \& Responsible AI](#securityresponsibleai) +- [Security & Responsible AI](#security--responsible-ai) - [License](#license) - [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) @@ -45,7 +45,7 @@ ## Experimental Technology Disclaimer -Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We’re building it in the open with the community and welcome: +Codex CLI is an experimental project under active development. It is not yet stable, may contain bugs, incomplete features, or undergo breaking changes. We're building it in the open with the community and welcome: - Bug reports - Feature requests @@ -115,59 +115,59 @@ codex "explain this codebase to me" codex --approval-mode full-auto "create the fanciest todo-list app" ``` -That’s it – Codex will scaffold a file, run it inside a sandbox, install any +That's it - Codex will scaffold a file, run it inside a sandbox, install any missing dependencies, and show you the live result. Approve the changes and -they’ll be committed to your working directory. +they'll be committed to your working directory. --- -## Why Codex? +## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want -ChatGPT‑level reasoning **plus** the power to actually run code, manipulate -files, and iterate – all under version control. In short, it’s _chat‑driven +ChatGPT-level reasoning **plus** the power to actually run code, manipulate +files, and iterate - all under version control. In short, it's _chat-driven development_ that understands and executes your repo. -- **Zero setup** — bring your OpenAI API key and it just works! +- **Zero setup** - bring your OpenAI API key and it just works! - **Full auto-approval, while safe + secure** by running network-disabled and directory-sandboxed -- **Multimodal** — pass in screenshots or diagrams to implement features ✨ +- **Multimodal** - pass in screenshots or diagrams to implement features ✨ And it's **fully open-source** so you can see and contribute to how it develops! --- -## Security Model & Permissions +## Security Model & Permissions Codex lets you decide _how much autonomy_ the agent receives and auto-approval policy via the `--approval-mode` flag (or the interactive onboarding prompt): -| Mode | What the agent may do without asking | Still requires approval | -| ------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Suggest**
    (default) | • Read any file in the repo | • **All** file writes/patches
    • **Any** arbitrary shell commands (aside from reading files) | -| **Auto Edit** | • Read **and** apply‑patch writes to files | • **All** shell commands | -| **Full Auto** | • Read/write files
    • Execute shell commands (network disabled, writes limited to your workdir) | – | +| Mode | What the agent may do without asking | Still requires approval | +| ------------------------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Suggest**
    (default) |
  • Read any file in the repo |
  • **All** file writes/patches
  • **Any** arbitrary shell commands (aside from reading files) | +| **Auto Edit** |
  • Read **and** apply-patch writes to files |
  • **All** shell commands | +| **Full Auto** |
  • Read/write files
  • Execute shell commands (network disabled, writes limited to your workdir) | - | -In **Full Auto** every command is run **network‑disabled** and confined to the -current working directory (plus temporary files) for defense‑in‑depth. Codex -will also show a warning/confirmation if you start in **auto‑edit** or -**full‑auto** while the directory is _not_ tracked by Git, so you always have a +In **Full Auto** every command is run **network-disabled** and confined to the +current working directory (plus temporary files) for defense-in-depth. Codex +will also show a warning/confirmation if you start in **auto-edit** or +**full-auto** while the directory is _not_ tracked by Git, so you always have a safety net. -Coming soon: you’ll be able to whitelist specific commands to auto‑execute with -the network enabled, once we’re confident in additional safeguards. +Coming soon: you'll be able to whitelist specific commands to auto-execute with +the network enabled, once we're confident in additional safeguards. ### Platform sandboxing details The hardening mechanism Codex uses depends on your OS: -- **macOS 12+** – commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). +- **macOS 12+** - commands are wrapped with **Apple Seatbelt** (`sandbox-exec`). - - Everything is placed in a read‑only jail except for a small set of + - Everything is placed in a read-only jail except for a small set of writable roots (`$PWD`, `$TMPDIR`, `~/.codex`, etc.). - - Outbound network is _fully blocked_ by default – even if a child process + - Outbound network is _fully blocked_ by default - even if a child process tries to `curl` somewhere it will fail. -- **Linux** – there is no sandboxing by default. +- **Linux** - there is no sandboxing by default. We recommend using Docker for sandboxing, where Codex launches itself inside a **minimal container image** and mounts your repo _read/write_ at the same path. A custom `iptables`/`ipset` firewall script denies all egress except the @@ -176,47 +176,47 @@ The hardening mechanism Codex uses depends on your OS: --- -## System Requirements +## System Requirements | Requirement | Details | | --------------------------- | --------------------------------------------------------------- | -| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | | Node.js | **22 or newer** (LTS recommended) | -| Git (optional, recommended) | 2.23+ for built‑in PR helpers | -| RAM | 4‑GB minimum (8‑GB recommended) | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | > Never run `sudo npm install -g`; fix npm permissions instead. --- -## CLI Reference +## CLI Reference | Command | Purpose | Example | | ------------------------------------ | ----------------------------------- | ------------------------------------ | | `codex` | Interactive REPL | `codex` | -| `codex "…"` | Initial prompt for interactive REPL | `codex "fix lint errors"` | -| `codex -q "…"` | Non‑interactive "quiet mode" | `codex -q --json "explain utils.ts"` | +| `codex "..."` | Initial prompt for interactive REPL | `codex "fix lint errors"` | +| `codex -q "..."` | Non-interactive "quiet mode" | `codex -q --json "explain utils.ts"` | | `codex completion ` | Print shell completion script | `codex completion bash` | Key flags: `--model/-m`, `--approval-mode/-a`, `--quiet/-q`, and `--notify`. --- -## Memory & Project Docs +## Memory & Project Docs Codex merges Markdown instructions in this order: -1. `~/.codex/instructions.md` – personal global guidance -2. `codex.md` at repo root – shared project notes -3. `codex.md` in cwd – sub‑package specifics +1. `~/.codex/instructions.md` - personal global guidance +2. `codex.md` at repo root - shared project notes +3. `codex.md` in cwd - sub-package specifics Disable with `--no-project-doc` or `CODEX_DISABLE_PROJECT_DOC=1`. --- -## Non‑interactive / CI mode +## Non-interactive / CI mode -Run Codex head‑less in pipelines. Example GitHub Action step: +Run Codex head-less in pipelines. Example GitHub Action step: ```yaml - name: Update changelog via Codex @@ -240,15 +240,15 @@ DEBUG=true codex ## Recipes -Below are a few bite‑size examples you can copy‑paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. +Below are a few bite-size examples you can copy-paste. Replace the text in quotes with your own task. See the [prompting guide](https://github.com/openai/codex/blob/main/codex-cli/examples/prompting_guide.md) for more tips and usage patterns. | ✨ | What you type | What happens | | --- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | +| 1 | `codex "Refactor the Dashboard component to React Hooks"` | Codex rewrites the class component, runs `npm test`, and shows the diff. | | 2 | `codex "Generate SQL migrations for adding a users table"` | Infers your ORM, creates migration files, and runs them in a sandboxed DB. | | 3 | `codex "Write unit tests for utils/date.ts"` | Generates tests, executes them, and iterates until they pass. | -| 4 | `codex "Bulk‑rename *.jpeg → *.jpg with git mv"` | Safely renames files and updates imports/usages. | -| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step‑by‑step human explanation. | +| 4 | `codex "Bulk-rename *.jpeg -> *.jpg with git mv"` | Safely renames files and updates imports/usages. | +| 5 | `codex "Explain what this regex does: ^(?=.*[A-Z]).{8,}$"` | Outputs a step-by-step human explanation. | | 6 | `codex "Carefully review this repo, and propose 3 high impact well-scoped PRs"` | Suggests impactful PRs in the current codebase. | | 7 | `codex "Look for vulnerabilities and create a security review report"` | Finds and explains security bugs. | @@ -257,7 +257,7 @@ Below are a few bite‑size examples you can copy‑paste. Replace the text in q ## Installation
    -From npm (Recommended) +From npm (Recommended) ```bash npm install -g @openai/codex @@ -272,7 +272,7 @@ pnpm add -g @openai/codex
    -Build from source +Build from source ```bash # Clone the repository and navigate to the CLI package @@ -289,7 +289,7 @@ pnpm build # Get the usage and the options node ./dist/cli.js --help -# Run the locally‑built CLI directly +# Run the locally-built CLI directly node ./dist/cli.js # Or link the command globally for convenience @@ -363,7 +363,7 @@ Codex runs model-generated commands in a sandbox. If a proposed command or file
    Does it work on Windows? -Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) – Codex has been tested on macOS and Linux with Node ≥ 22. +Not directly. It requires [Windows Subsystem for Linux (WSL2)](https://learn.microsoft.com/en-us/windows/wsl/install) - Codex has been tested on macOS and Linux with Node 22.
    @@ -394,12 +394,12 @@ OpenAI rejected the request. Error details: Status: 400, Code: unsupported_param ## Funding Opportunity -We’re excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. - Grants are awarded in **$25,000** API credit increments. - Applications are reviewed **on a rolling basis**. -**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** --- @@ -407,14 +407,14 @@ We’re excited to launch a **$1 million initiative** supporting open source pr This project is under active development and the code will likely change pretty significantly. We'll update this message once that's complete! -More broadly we welcome contributions – whether you are opening your very first pull request or you’re a seasoned maintainer. At the same time we care about reliability and long‑term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what “high‑quality” means in practice and should make the whole process transparent and friendly. +More broadly we welcome contributions - whether you are opening your very first pull request or you're a seasoned maintainer. At the same time we care about reliability and long-term maintainability, so the bar for merging code is intentionally **high**. The guidelines below spell out what "high-quality" means in practice and should make the whole process transparent and friendly. ### Development workflow -- Create a _topic branch_ from `main` – e.g. `feat/interactive-prompt`. +- Create a _topic branch_ from `main` - e.g. `feat/interactive-prompt`. - Keep your changes focused. Multiple unrelated fixes should be opened as separate PRs. -- Use `pnpm test:watch` during development for super‑fast feedback. -- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type‑checking. +- Use `pnpm test:watch` during development for super-fast feedback. +- We use **Vitest** for unit tests, **ESLint** + **Prettier** for style, and **TypeScript** for type-checking. - Before pushing, run the full test/type/lint suite: ### Git Hooks with Husky @@ -436,16 +436,16 @@ npm test && npm run lint && npm run typecheck I have read the CLA Document and I hereby sign the CLA ``` - The CLA‑Assistant bot will turn the PR status green once all authors have signed. + The CLA-Assistant bot will turn the PR status green once all authors have signed. ```bash -# Watch mode (tests rerun on change) +# Watch mode (tests rerun on change) pnpm test:watch -# Type‑check without emitting files +# Type-check without emitting files pnpm typecheck -# Automatically fix lint + prettier issues +# Automatically fix lint + prettier issues pnpm lint:fix pnpm format:fix ``` @@ -475,35 +475,35 @@ Run the CLI via the flake app: nix run .#codex ``` -### Writing high‑impact code changes +### Writing high-impact code changes 1. **Start with an issue.** Open a new one or comment on an existing discussion so we can agree on the solution before code is written. -2. **Add or update tests.** Every new feature or bug‑fix should come with test coverage that fails before your change and passes afterwards. 100 % coverage is not required, but aim for meaningful assertions. -3. **Document behaviour.** If your change affects user‑facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. +2. **Add or update tests.** Every new feature or bug-fix should come with test coverage that fails before your change and passes afterwards. 100% coverage is not required, but aim for meaningful assertions. +3. **Document behaviour.** If your change affects user-facing behaviour, update the README, inline help (`codex --help`), or relevant example projects. 4. **Keep commits atomic.** Each commit should compile and the tests should pass. This makes reviews and potential rollbacks easier. ### Opening a pull request -- Fill in the PR template (or include similar information) – **What? Why? How?** +- Fill in the PR template (or include similar information) - **What? Why? How?** - Run **all** checks locally (`npm test && npm run lint && npm run typecheck`). CI failures that could have been caught locally slow down the process. -- Make sure your branch is up‑to‑date with `main` and that you have resolved merge conflicts. -- Mark the PR as **Ready for review** only when you believe it is in a merge‑able state. +- Make sure your branch is up-to-date with `main` and that you have resolved merge conflicts. +- Mark the PR as **Ready for review** only when you believe it is in a merge-able state. ### Review process 1. One maintainer will be assigned as a primary reviewer. -2. We may ask for changes – please do not take this personally. We value the work, we just also value consistency and long‑term maintainability. -3. When there is consensus that the PR meets the bar, a maintainer will squash‑and‑merge. +2. We may ask for changes - please do not take this personally. We value the work, we just also value consistency and long-term maintainability. +3. When there is consensus that the PR meets the bar, a maintainer will squash-and-merge. ### Community values - **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). -- **Assume good intent.** Written communication is hard – err on the side of generosity. +- **Assume good intent.** Written communication is hard - err on the side of generosity. - **Teach & learn.** If you spot something confusing, open an issue or PR with improvements. ### Getting help -If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ – please open a Discussion or jump into the relevant issue. We are happy to help. +If you run into problems setting up the project, would like feedback on an idea, or just want to say _hi_ - please open a Discussion or jump into the relevant issue. We are happy to help. Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: @@ -512,22 +512,21 @@ Together we can make Codex CLI an incredible tool. **Happy hacking!** :rocket: All contributors **must** accept the CLA. The process is lightweight: 1. Open your pull request. -2. Paste the following comment (or reply `recheck` if you’ve signed before): +2. Paste the following comment (or reply `recheck` if you've signed before): ```text I have read the CLA Document and I hereby sign the CLA ``` -3. The CLA‑Assistant bot records your signature in the repo and marks the status check as passed. +3. The CLA-Assistant bot records your signature in the repo and marks the status check as passed. No special Git commands, email attachments, or commit footers required. #### Quick fixes -| Scenario | Command | -| ----------------- | ----------------------------------------------------------------------------------------- | -| Amend last commit | `git commit --amend -s --no-edit && git push -f` | -| GitHub UI only | Edit the commit message in the PR → add
    `Signed-off-by: Your Name ` | +| Scenario | Command | +| ----------------- | ------------------------------------------------ | +| Amend last commit | `git commit --amend -s --no-edit && git push -f` | The **DCO check** blocks merges until every commit in the PR carries the footer (with squash this is just the one). @@ -548,12 +547,12 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c --- -## Security & Responsible AI +## Security & Responsible AI -Have you discovered a vulnerability or have concerns about model output? Please e‑mail **security@openai.com** and we will respond promptly. +Have you discovered a vulnerability or have concerns about model output? Please e-mail **security@openai.com** and we will respond promptly. --- ## License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100755 index 0000000000..812d1c6b69 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) From df973e700762a330e593f730b6e19c36861c247a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 09:21:01 -0700 Subject: [PATCH 48/84] add check to ensure ToC in README.md matches headings in the file --- .github/workflows/ci.yml | 2 + README.md | 8 ++- scripts/readme_toc.py | 119 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 scripts/readme_toc.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f5dd0d31a..508b5b9bd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,3 +70,5 @@ jobs: - name: Ensure README.md contains only ASCII and certain Unicode code points run: ./scripts/asciicheck.py README.md + - name: Check README ToC + run: python3 scripts/readme_toc.py README.md diff --git a/README.md b/README.md index 9ac89499ac..ed8d3ff84c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@
    Table of Contents + + - [Experimental Technology Disclaimer](#experimental-technology-disclaimer) - [Quickstart](#quickstart) - [Why Codex?](#why-codex) @@ -19,13 +21,16 @@ - [CLI Reference](#cli-reference) - [Memory & Project Docs](#memory--project-docs) - [Non-interactive / CI mode](#non-interactive--ci-mode) +- [Tracing / Verbose Logging](#tracing--verbose-logging) - [Recipes](#recipes) - [Installation](#installation) - [Configuration](#configuration) - [FAQ](#faq) +- [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) - [Funding Opportunity](#funding-opportunity) - [Contributing](#contributing) - [Development workflow](#development-workflow) + - [Git Hooks with Husky](#git-hooks-with-husky) - [Nix Flake Development](#nix-flake-development) - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) @@ -37,7 +42,8 @@ - [Releasing `codex`](#releasing-codex) - [Security & Responsible AI](#security--responsible-ai) - [License](#license) -- [Zero Data Retention (ZDR) Organization Limitation](#zero-data-retention-zdr-organization-limitation) + +
    diff --git a/scripts/readme_toc.py b/scripts/readme_toc.py new file mode 100755 index 0000000000..fb1ac066a7 --- /dev/null +++ b/scripts/readme_toc.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +""" +Utility script to verify (and optionally fix) the Table of Contents in a +Markdown file. By default, it checks that the ToC between `` +and `` matches the headings in the file. With --fix, it +rewrites the file to update the ToC. +""" + +import argparse +import sys +import re +import difflib +from pathlib import Path +from typing import List + +# Markers for the Table of Contents section +BEGIN_TOC: str = "" +END_TOC: str = "" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check and optionally fix the README.md Table of Contents." + ) + parser.add_argument( + "file", nargs="?", default="README.md", help="Markdown file to process" + ) + parser.add_argument( + "--fix", action="store_true", help="Rewrite file with updated ToC" + ) + args = parser.parse_args() + path = Path(args.file) + return check_or_fix(path, args.fix) + + +def generate_toc_lines(content: str) -> List[str]: + """ + Generate markdown list lines for headings (## to ######) in content. + """ + lines = content.splitlines() + headings = [] + in_code = False + for line in lines: + if line.strip().startswith("```"): + in_code = not in_code + continue + if in_code: + continue + m = re.match(r"^(#{2,6})\s+(.*)$", line) + if not m: + continue + level = len(m.group(1)) + text = m.group(2).strip() + headings.append((level, text)) + + toc = [] + for level, text in headings: + indent = " " * (level - 2) + slug = text.lower() + # normalize spaces and dashes + slug = slug.replace("\u00a0", " ") + slug = slug.replace("\u2011", "-").replace("\u2013", "-").replace("\u2014", "-") + # drop other punctuation + slug = re.sub(r"[^0-9a-z\s-]", "", slug) + slug = slug.strip().replace(" ", "-") + toc.append(f"{indent}- [{text}](#{slug})") + return toc + + +def check_or_fix(readme_path: Path, fix: bool) -> int: + if not readme_path.is_file(): + print(f"Error: file not found: {readme_path}", file=sys.stderr) + return 1 + content = readme_path.read_text(encoding="utf-8") + lines = content.splitlines() + # locate ToC markers + try: + begin_idx = next(i for i, l in enumerate(lines) if l.strip() == BEGIN_TOC) + end_idx = next(i for i, l in enumerate(lines) if l.strip() == END_TOC) + except StopIteration: + print( + f"Error: Could not locate '{BEGIN_TOC}' or '{END_TOC}' in {readme_path}.", + file=sys.stderr, + ) + return 1 + # extract current ToC list items + current_block = lines[begin_idx + 1 : end_idx] + current = [l for l in current_block if l.lstrip().startswith("- [")] + # generate expected ToC + expected = generate_toc_lines(content) + if current == expected: + return 0 + if not fix: + print( + "ERROR: README ToC is out of date. Diff between existing and generated ToC:" + ) + # Show full unified diff of current vs expected + diff = difflib.unified_diff( + current, + expected, + fromfile="existing ToC", + tofile="generated ToC", + lineterm="", + ) + for line in diff: + print(line) + return 1 + # rebuild file with updated ToC + prefix = lines[: begin_idx + 1] + suffix = lines[end_idx:] + new_lines = prefix + [""] + expected + [""] + suffix + readme_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8") + print(f"Updated ToC in {readme_path}.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From debc389f94f0a3d02ffa848b13c774448680d8c6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 09:40:21 -0700 Subject: [PATCH 49/84] add instructions for connecting to a visual debugger under Contributing --- README.md | 61 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ed8d3ff84c..ab4701bf7d 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ - [Contributing](#contributing) - [Development workflow](#development-workflow) - [Git Hooks with Husky](#git-hooks-with-husky) - - [Nix Flake Development](#nix-flake-development) + - [Debugging](#debugging) - [Writing high-impact code changes](#writing-high-impact-code-changes) - [Opening a pull request](#opening-a-pull-request) - [Review process](#review-process) @@ -40,6 +40,8 @@ - [Contributor License Agreement (CLA)](#contributor-license-agreement-cla) - [Quick fixes](#quick-fixes) - [Releasing `codex`](#releasing-codex) + - [Alternative Build Options](#alternative-build-options) + - [Nix Flake Development](#nix-flake-development) - [Security & Responsible AI](#security--responsible-ai) - [License](#license) @@ -433,7 +435,7 @@ This project uses [Husky](https://typicode.github.io/husky/) to enforce code qua These hooks help maintain code quality and prevent pushing code with failing tests. For more details, see [HUSKY.md](./codex-cli/HUSKY.md). ```bash -npm test && npm run lint && npm run typecheck +pnpm test && pnpm run lint && pnpm run typecheck ``` - If you have **not** yet signed the Contributor License Agreement (CLA), add a PR comment containing the exact text @@ -456,30 +458,14 @@ pnpm lint:fix pnpm format:fix ``` -#### Nix Flake Development +### Debugging -Prerequisite: Nix >= 2.4 with flakes enabled (`experimental-features = nix-command flakes` in `~/.config/nix/nix.conf`). +To debug the CLI with a visual debugger, do the following in the `codex-cli` folder: -Enter a Nix development shell: - -```bash -nix develop -``` - -This shell includes Node.js, installs dependencies, builds the CLI, and provides a `codex` command alias. - -Build and run the CLI directly: - -```bash -nix build -./result/bin/codex --help -``` - -Run the CLI via the flake app: - -```bash -nix run .#codex -``` +- Run `pnpm run build` to build the CLI, which will generate `cli.js.map` alongside `cli.js` in the `dist` folder. +- Run the CLI with `node --inspect-brk ./dist/cli.js` The program then waits until a debugger is attached before proceeding. Options: + - In VS Code, choose **Debug: Attach to Node Process** from the command palette and choose the option in the dropdown with debug port `9229` (likely the first option) + - Go to in Chrome and find **localhost:9229** and click **trace** ### Writing high-impact code changes @@ -551,6 +537,33 @@ To publish a new version of the CLI, run the release scripts defined in `codex-c 5. Copy README, build, and publish to npm: `pnpm release` 6. Push to branch: `git push origin HEAD` +### Alternative Build Options + +#### Nix Flake Development + +Prerequisite: Nix >= 2.4 with flakes enabled (`experimental-features = nix-command flakes` in `~/.config/nix/nix.conf`). + +Enter a Nix development shell: + +```bash +nix develop +``` + +This shell includes Node.js, installs dependencies, builds the CLI, and provides a `codex` command alias. + +Build and run the CLI directly: + +```bash +nix build +./result/bin/codex --help +``` + +Run the CLI via the flake app: + +```bash +nix run .#codex +``` + --- ## Security & Responsible AI From 012ad7e69ebcf2d46a4a6ee17f6bac8cc492f624 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 12:56:07 -0700 Subject: [PATCH 50/84] when a shell tool call invokes apply_patch, resolve relative paths against workdir, if specified --- codex-cli/src/approvals.ts | 36 ++++++++++++++++--- .../src/utils/agent/handle-exec-command.ts | 4 +-- codex-cli/tests/approvals.test.ts | 8 ++++- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index ff37a8903f..e962d548df 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -71,13 +71,14 @@ export type ApprovalPolicy = */ export function canAutoApprove( command: ReadonlyArray, + workdir: string | undefined, policy: ApprovalPolicy, writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { if (command[0] === "apply_patch") { return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + ? canAutoApproveApplyPatch(command[1], workdir, writableRoots, policy) : { type: "reject", reason: "Invalid apply_patch command", @@ -103,7 +104,12 @@ export function canAutoApprove( ) { const applyPatchArg = tryParseApplyPatch(command[2]); if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); + return canAutoApproveApplyPatch( + applyPatchArg, + workdir, + writableRoots, + policy, + ); } let bashCmd; @@ -162,6 +168,7 @@ export function canAutoApprove( function canAutoApproveApplyPatch( applyPatchArg: string, + workdir: string | undefined, writableRoots: ReadonlyArray, policy: ApprovalPolicy, ): SafetyAssessment { @@ -179,7 +186,13 @@ function canAutoApproveApplyPatch( break; } - if (isWritePatchConstrainedToWritablePaths(applyPatchArg, writableRoots)) { + if ( + isWritePatchConstrainedToWritablePaths( + applyPatchArg, + workdir, + writableRoots, + ) + ) { return { type: "auto-approve", reason: "apply_patch command is constrained to writable paths", @@ -208,6 +221,7 @@ function canAutoApproveApplyPatch( */ function isWritePatchConstrainedToWritablePaths( applyPatchArg: string, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { // `identify_files_needed()` returns a list of files that will be modified or @@ -222,10 +236,12 @@ function isWritePatchConstrainedToWritablePaths( return ( allPathsConstrainedTowritablePaths( identify_files_needed(applyPatchArg), + workdir, writableRoots, ) && allPathsConstrainedTowritablePaths( identify_files_added(applyPatchArg), + workdir, writableRoots, ) ); @@ -233,19 +249,29 @@ function isWritePatchConstrainedToWritablePaths( function allPathsConstrainedTowritablePaths( candidatePaths: ReadonlyArray, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { return candidatePaths.every((candidatePath) => - isPathConstrainedTowritablePaths(candidatePath, writableRoots), + isPathConstrainedTowritablePaths(candidatePath, workdir, writableRoots), ); } /** If candidatePath is relative, it will be resolved against cwd. */ function isPathConstrainedTowritablePaths( candidatePath: string, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { - const candidateAbsolutePath = path.resolve(candidatePath); + let candidateAbsolutePath: string; + if (path.isAbsolute(candidatePath)) { + candidateAbsolutePath = candidatePath; + } else if (workdir != null) { + candidateAbsolutePath = path.resolve(workdir, candidatePath); + } else { + candidateAbsolutePath = path.resolve(candidatePath); + } + return writableRoots.some((writablePath) => pathContains(writablePath, candidateAbsolutePath), ); diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index aea2c3a707..1932ab9840 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -81,7 +81,7 @@ export async function handleExecCommand( ) => Promise, abortSignal?: AbortSignal, ): Promise { - const { cmd: command } = args; + const { cmd: command, workdir } = args; const key = deriveCommandKey(command); @@ -103,7 +103,7 @@ export async function handleExecCommand( // working directory so that edits are constrained to the project root. If // the caller wishes to broaden or restrict the set it can be made // configurable in the future. - const safety = canAutoApprove(command, policy, [process.cwd()]); + const safety = canAutoApprove(command, workdir, policy, [process.cwd()]); let runInSandbox: boolean; switch (safety.type) { diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index a90abad6eb..94daacce00 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -11,7 +11,13 @@ describe("canAutoApprove()", () => { const writeablePaths: Array = []; const check = (command: ReadonlyArray): SafetyAssessment => - canAutoApprove(command, "suggest", writeablePaths, env); + canAutoApprove( + command, + /* workdir */ undefined, + "suggest", + writeablePaths, + env, + ); test("simple safe commands", () => { expect(check(["ls"])).toEqual({ From 9c51116b01fbc15c67f6cd69f5a19e3119bc2d3c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 22 Apr 2025 12:56:07 -0700 Subject: [PATCH 51/84] when a shell tool call invokes apply_patch, resolve relative paths against workdir, if specified --- codex-cli/src/approvals.ts | 49 +++++++++++++++++-- codex-cli/src/utils/agent/exec.ts | 13 +++-- .../src/utils/agent/handle-exec-command.ts | 6 +-- codex-cli/tests/approvals.test.ts | 8 ++- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/codex-cli/src/approvals.ts b/codex-cli/src/approvals.ts index ff37a8903f..5ea73eab07 100644 --- a/codex-cli/src/approvals.ts +++ b/codex-cli/src/approvals.ts @@ -71,13 +71,14 @@ export type ApprovalPolicy = */ export function canAutoApprove( command: ReadonlyArray, + workdir: string | undefined, policy: ApprovalPolicy, writableRoots: ReadonlyArray, env: NodeJS.ProcessEnv = process.env, ): SafetyAssessment { if (command[0] === "apply_patch") { return command.length === 2 && typeof command[1] === "string" - ? canAutoApproveApplyPatch(command[1], writableRoots, policy) + ? canAutoApproveApplyPatch(command[1], workdir, writableRoots, policy) : { type: "reject", reason: "Invalid apply_patch command", @@ -103,7 +104,12 @@ export function canAutoApprove( ) { const applyPatchArg = tryParseApplyPatch(command[2]); if (applyPatchArg != null) { - return canAutoApproveApplyPatch(applyPatchArg, writableRoots, policy); + return canAutoApproveApplyPatch( + applyPatchArg, + workdir, + writableRoots, + policy, + ); } let bashCmd; @@ -162,6 +168,7 @@ export function canAutoApprove( function canAutoApproveApplyPatch( applyPatchArg: string, + workdir: string | undefined, writableRoots: ReadonlyArray, policy: ApprovalPolicy, ): SafetyAssessment { @@ -179,7 +186,13 @@ function canAutoApproveApplyPatch( break; } - if (isWritePatchConstrainedToWritablePaths(applyPatchArg, writableRoots)) { + if ( + isWritePatchConstrainedToWritablePaths( + applyPatchArg, + workdir, + writableRoots, + ) + ) { return { type: "auto-approve", reason: "apply_patch command is constrained to writable paths", @@ -208,6 +221,7 @@ function canAutoApproveApplyPatch( */ function isWritePatchConstrainedToWritablePaths( applyPatchArg: string, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { // `identify_files_needed()` returns a list of files that will be modified or @@ -222,10 +236,12 @@ function isWritePatchConstrainedToWritablePaths( return ( allPathsConstrainedTowritablePaths( identify_files_needed(applyPatchArg), + workdir, writableRoots, ) && allPathsConstrainedTowritablePaths( identify_files_added(applyPatchArg), + workdir, writableRoots, ) ); @@ -233,24 +249,47 @@ function isWritePatchConstrainedToWritablePaths( function allPathsConstrainedTowritablePaths( candidatePaths: ReadonlyArray, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { return candidatePaths.every((candidatePath) => - isPathConstrainedTowritablePaths(candidatePath, writableRoots), + isPathConstrainedTowritablePaths(candidatePath, workdir, writableRoots), ); } /** If candidatePath is relative, it will be resolved against cwd. */ function isPathConstrainedTowritablePaths( candidatePath: string, + workdir: string | undefined, writableRoots: ReadonlyArray, ): boolean { - const candidateAbsolutePath = path.resolve(candidatePath); + const candidateAbsolutePath = resolvePathAgainstWorkdir( + candidatePath, + workdir, + ); + return writableRoots.some((writablePath) => pathContains(writablePath, candidateAbsolutePath), ); } +/** + * If not already an absolute path, resolves `candidatePath` against `workdir` + * if specified; otherwise, against `process.cwd()`. + */ +export function resolvePathAgainstWorkdir( + candidatePath: string, + workdir: string | undefined, +): string { + if (path.isAbsolute(candidatePath)) { + return candidatePath; + } else if (workdir != null) { + return path.resolve(workdir, candidatePath); + } else { + return path.resolve(candidatePath); + } +} + /** Both `parent` and `child` must be absolute paths. */ function pathContains(parent: string, child: string): boolean { const relative = path.relative(parent, child); diff --git a/codex-cli/src/utils/agent/exec.ts b/codex-cli/src/utils/agent/exec.ts index 25c6f86abc..f0177979ec 100644 --- a/codex-cli/src/utils/agent/exec.ts +++ b/codex-cli/src/utils/agent/exec.ts @@ -10,6 +10,7 @@ import { formatCommandForDisplay } from "../../format-command.js"; import fs from "fs"; import os from "os"; import { parse } from "shell-quote"; +import { resolvePathAgainstWorkdir } from "src/approvals.js"; const DEFAULT_TIMEOUT_MS = 10_000; // 10 seconds @@ -60,16 +61,20 @@ export function exec( return execForSandbox(cmd, opts, writableRoots, abortSignal); } -export function execApplyPatch(patchText: string): ExecResult { +export function execApplyPatch( + patchText: string, + workdir: string | undefined, +): ExecResult { // This is a temporary measure to understand what are the common base commands // until we start persisting and uploading rollouts try { const result = process_patch( patchText, - (p) => fs.readFileSync(p, "utf8"), - (p, c) => fs.writeFileSync(p, c, "utf8"), - (p) => fs.unlinkSync(p), + (p) => fs.readFileSync(resolvePathAgainstWorkdir(p, workdir), "utf8"), + (p, c) => + fs.writeFileSync(resolvePathAgainstWorkdir(p, workdir), c, "utf8"), + (p) => fs.unlinkSync(resolvePathAgainstWorkdir(p, workdir)), ); return { stdout: result, diff --git a/codex-cli/src/utils/agent/handle-exec-command.ts b/codex-cli/src/utils/agent/handle-exec-command.ts index aea2c3a707..85d6869192 100644 --- a/codex-cli/src/utils/agent/handle-exec-command.ts +++ b/codex-cli/src/utils/agent/handle-exec-command.ts @@ -81,7 +81,7 @@ export async function handleExecCommand( ) => Promise, abortSignal?: AbortSignal, ): Promise { - const { cmd: command } = args; + const { cmd: command, workdir } = args; const key = deriveCommandKey(command); @@ -103,7 +103,7 @@ export async function handleExecCommand( // working directory so that edits are constrained to the project root. If // the caller wishes to broaden or restrict the set it can be made // configurable in the future. - const safety = canAutoApprove(command, policy, [process.cwd()]); + const safety = canAutoApprove(command, workdir, policy, [process.cwd()]); let runInSandbox: boolean; switch (safety.type) { @@ -247,7 +247,7 @@ async function execCommand( const start = Date.now(); const execResult = applyPatchCommand != null - ? execApplyPatch(applyPatchCommand.patch) + ? execApplyPatch(applyPatchCommand.patch, workdir) : await exec( { ...execInput, additionalWritableRoots }, await getSandbox(runInSandbox), diff --git a/codex-cli/tests/approvals.test.ts b/codex-cli/tests/approvals.test.ts index a90abad6eb..94daacce00 100644 --- a/codex-cli/tests/approvals.test.ts +++ b/codex-cli/tests/approvals.test.ts @@ -11,7 +11,13 @@ describe("canAutoApprove()", () => { const writeablePaths: Array = []; const check = (command: ReadonlyArray): SafetyAssessment => - canAutoApprove(command, "suggest", writeablePaths, env); + canAutoApprove( + command, + /* workdir */ undefined, + "suggest", + writeablePaths, + env, + ); test("simple safe commands", () => { expect(check(["ls"])).toEqual({ From 05aa1666737d18d1267d085aac4856a822960c71 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 13:15:13 -0700 Subject: [PATCH 52/84] fix: do not grant "node" user sudo access when using run_in_container.sh --- codex-cli/Dockerfile | 5 +---- codex-cli/scripts/run_in_container.sh | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/codex-cli/Dockerfile b/codex-cli/Dockerfile index 5f89420372..ef1ccfba90 100644 --- a/codex-cli/Dockerfile +++ b/codex-cli/Dockerfile @@ -20,7 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ less \ man-db \ procps \ - sudo \ unzip \ ripgrep \ zsh \ @@ -50,7 +49,5 @@ RUN npm install -g codex.tgz \ # Copy and set up firewall script COPY scripts/init_firewall.sh /usr/local/bin/ USER root -RUN chmod +x /usr/local/bin/init_firewall.sh && \ - echo "node ALL=(root) NOPASSWD: /usr/local/bin/init_firewall.sh" > /etc/sudoers.d/node-firewall && \ - chmod 0440 /etc/sudoers.d/node-firewall +RUN chmod +x /usr/local/bin/init_firewall.sh USER node diff --git a/codex-cli/scripts/run_in_container.sh b/codex-cli/scripts/run_in_container.sh index c95c57aead..b062e85964 100755 --- a/codex-cli/scripts/run_in_container.sh +++ b/codex-cli/scripts/run_in_container.sh @@ -57,8 +57,10 @@ docker run --name "$CONTAINER_NAME" -d \ codex \ sleep infinity -# Initialize the firewall inside the container. -docker exec "$CONTAINER_NAME" bash -c "sudo /usr/local/bin/init_firewall.sh" +# Initialize the firewall inside the container with root privileges. We avoid +# using sudo inside the container altogether by invoking the script directly +# as the root user via docker exec. +docker exec --user root "$CONTAINER_NAME" /usr/local/bin/init_firewall.sh # Execute the provided command in the container, ensuring it runs in the work directory. # We use a parameterized bash command to safely handle the command and directory. From 65d1cc097d8d6342a9e1a2e468d808a497b8ac7b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 14:15:59 -0700 Subject: [PATCH 53/84] fix: do not grant "node" user sudo access when using run_in_container.sh --- codex-cli/Dockerfile | 11 +++++------ codex-cli/scripts/run_in_container.sh | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-cli/Dockerfile b/codex-cli/Dockerfile index 5f89420372..4ed3089bbb 100644 --- a/codex-cli/Dockerfile +++ b/codex-cli/Dockerfile @@ -20,7 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ less \ man-db \ procps \ - sudo \ unzip \ ripgrep \ zsh \ @@ -47,10 +46,10 @@ RUN npm install -g codex.tgz \ && rm -rf /usr/local/share/npm-global/lib/node_modules/codex-cli/tests \ && rm -rf /usr/local/share/npm-global/lib/node_modules/codex-cli/docs -# Copy and set up firewall script -COPY scripts/init_firewall.sh /usr/local/bin/ +# Copy and set up firewall script as root. USER root -RUN chmod +x /usr/local/bin/init_firewall.sh && \ - echo "node ALL=(root) NOPASSWD: /usr/local/bin/init_firewall.sh" > /etc/sudoers.d/node-firewall && \ - chmod 0440 /etc/sudoers.d/node-firewall +COPY scripts/init_firewall.sh /usr/local/bin/ +RUN chmod 500 /usr/local/bin/init_firewall.sh + +# Drop back to non-root. USER node diff --git a/codex-cli/scripts/run_in_container.sh b/codex-cli/scripts/run_in_container.sh index c95c57aead..1da286a743 100755 --- a/codex-cli/scripts/run_in_container.sh +++ b/codex-cli/scripts/run_in_container.sh @@ -57,8 +57,8 @@ docker run --name "$CONTAINER_NAME" -d \ codex \ sleep infinity -# Initialize the firewall inside the container. -docker exec "$CONTAINER_NAME" bash -c "sudo /usr/local/bin/init_firewall.sh" +# Initialize the firewall inside the container with root privileges. +docker exec --user root "$CONTAINER_NAME" /usr/local/bin/init_firewall.sh # Execute the provided command in the container, ensuring it runs in the work directory. # We use a parameterized bash command to safely handle the command and directory. From 55a8e70c5dc2248b3d48239dd2ecc2849a6bbafd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 14:29:32 -0700 Subject: [PATCH 54/84] fix: update scripts/build_container.sh to use pnpm instead of npm --- codex-cli/scripts/build_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-cli/scripts/build_container.sh b/codex-cli/scripts/build_container.sh index fd4c8f5a86..d4d29f6b34 100755 --- a/codex-cli/scripts/build_container.sh +++ b/codex-cli/scripts/build_container.sh @@ -8,9 +8,9 @@ pushd "$SCRIPT_DIR/.." >> /dev/null || { echo "Error: Failed to change directory to $SCRIPT_DIR/.." exit 1 } -npm install -npm run build +pnpm install +pnpm run build rm -rf ./dist/openai-codex-*.tgz -npm pack --pack-destination ./dist +pnpm pack --pack-destination ./dist mv ./dist/openai-codex-*.tgz ./dist/codex.tgz docker build -t codex -f "./Dockerfile" . From 36987ab76c5986b35f0956ce26e603639421b393 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 15:03:56 -0700 Subject: [PATCH 55/84] feat: introduce codex_execpolicy crate for defining "safe" commands --- codex-rs/Cargo.lock | 1063 ++++++++++++++++- codex-rs/Cargo.toml | 1 + codex-rs/execpolicy/Cargo.toml | 28 + codex-rs/execpolicy/README.md | 180 +++ codex-rs/execpolicy/build.rs | 3 + codex-rs/execpolicy/src/arg_matcher.rs | 118 ++ codex-rs/execpolicy/src/arg_resolver.rs | 194 +++ codex-rs/execpolicy/src/arg_type.rs | 87 ++ codex-rs/execpolicy/src/default.policy | 202 ++++ codex-rs/execpolicy/src/error.rs | 96 ++ codex-rs/execpolicy/src/exec_call.rs | 28 + codex-rs/execpolicy/src/execv_checker.rs | 263 ++++ codex-rs/execpolicy/src/lib.rs | 45 + codex-rs/execpolicy/src/main.rs | 166 +++ codex-rs/execpolicy/src/opt.rs | 77 ++ codex-rs/execpolicy/src/policy.rs | 103 ++ codex-rs/execpolicy/src/policy_parser.rs | 222 ++++ codex-rs/execpolicy/src/program.rs | 247 ++++ codex-rs/execpolicy/src/sed_command.rs | 17 + codex-rs/execpolicy/src/valid_exec.rs | 95 ++ codex-rs/execpolicy/tests/bad.rs | 9 + codex-rs/execpolicy/tests/cp.rs | 85 ++ codex-rs/execpolicy/tests/good.rs | 9 + codex-rs/execpolicy/tests/head.rs | 132 ++ codex-rs/execpolicy/tests/literal.rs | 50 + codex-rs/execpolicy/tests/ls.rs | 166 +++ .../execpolicy/tests/parse_sed_command.rs | 23 + codex-rs/execpolicy/tests/pwd.rs | 85 ++ codex-rs/execpolicy/tests/sed.rs | 83 ++ 29 files changed, 3830 insertions(+), 47 deletions(-) create mode 100644 codex-rs/execpolicy/Cargo.toml create mode 100644 codex-rs/execpolicy/README.md create mode 100644 codex-rs/execpolicy/build.rs create mode 100644 codex-rs/execpolicy/src/arg_matcher.rs create mode 100644 codex-rs/execpolicy/src/arg_resolver.rs create mode 100644 codex-rs/execpolicy/src/arg_type.rs create mode 100644 codex-rs/execpolicy/src/default.policy create mode 100644 codex-rs/execpolicy/src/error.rs create mode 100644 codex-rs/execpolicy/src/exec_call.rs create mode 100644 codex-rs/execpolicy/src/execv_checker.rs create mode 100644 codex-rs/execpolicy/src/lib.rs create mode 100644 codex-rs/execpolicy/src/main.rs create mode 100644 codex-rs/execpolicy/src/opt.rs create mode 100644 codex-rs/execpolicy/src/policy.rs create mode 100644 codex-rs/execpolicy/src/policy_parser.rs create mode 100644 codex-rs/execpolicy/src/program.rs create mode 100644 codex-rs/execpolicy/src/sed_command.rs create mode 100644 codex-rs/execpolicy/src/valid_exec.rs create mode 100644 codex-rs/execpolicy/tests/bad.rs create mode 100644 codex-rs/execpolicy/tests/cp.rs create mode 100644 codex-rs/execpolicy/tests/good.rs create mode 100644 codex-rs/execpolicy/tests/head.rs create mode 100644 codex-rs/execpolicy/tests/literal.rs create mode 100644 codex-rs/execpolicy/tests/ls.rs create mode 100644 codex-rs/execpolicy/tests/parse_sed_command.rs create mode 100644 codex-rs/execpolicy/tests/pwd.rs create mode 100644 codex-rs/execpolicy/tests/sed.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f9f5860861..1f91c0072b 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "addr2line" version = "0.21.0" @@ -17,6 +27,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy 0.7.35", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -26,6 +48,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocative" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fac2ce611db8b8cee9b2aa886ca03c924e9da5e5295d0dbd0526e5d0b0710f7" +dependencies = [ + "allocative_derive", + "bumpalo", + "ctor", + "hashbrown 0.14.5", + "num-bigint", +] + +[[package]] +name = "allocative_derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -47,6 +93,15 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "ansi-to-tui" version = "7.0.0" @@ -128,6 +183,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -174,7 +238,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -222,6 +286,33 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.9.0" @@ -262,6 +353,12 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.10.1" @@ -298,6 +395,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.40" @@ -308,6 +411,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -331,7 +435,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", "terminal_size", ] @@ -344,7 +448,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -353,6 +457,21 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "clipboard-win" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmp_any" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9b18233253483ce2f65329a24072ec414db782531bdbb7d0bbc4bd2ce6b7e21" + [[package]] name = "codex-ansi-escape" version = "0.1.0" @@ -445,6 +564,26 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "codex-execpolicy" +version = "0.1.0" +dependencies = [ + "allocative", + "anyhow", + "clap", + "derive_more", + "env_logger", + "log", + "multimap", + "path-absolutize", + "regex", + "serde", + "serde_json", + "serde_with", + "starlark", + "tempfile", +] + [[package]] name = "codex-interactive" version = "0.1.0" @@ -551,6 +690,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -588,7 +736,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.9.0", "crossterm_winapi", "mio", "parking_lot", @@ -607,6 +755,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "darling" version = "0.20.11" @@ -627,8 +791,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.100", ] [[package]] @@ -639,7 +803,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -660,6 +824,17 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "debugserver-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf6834a70ed14e8e4e41882df27190bea150f1f6ecf461f1033f8739cd8af4a" +dependencies = [ + "schemafy", + "serde", + "serde_json", +] + [[package]] name = "deranged" version = "0.4.0" @@ -667,6 +842,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn 2.0.100", + "unicode-xid", ] [[package]] @@ -701,6 +910,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.5.0" @@ -713,6 +932,27 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users 0.4.6", + "winapi", +] + +[[package]] +name = "display_container" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a110a75c96bedec8e65823dea00a1d710288b7a369d95fd8a0f5127639466fa" +dependencies = [ + "either", + "indenter", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -721,7 +961,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -730,12 +970,41 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dupe" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed2bc011db9c93fbc2b6cdb341a53737a55bafb46dbb74cf6764fc33a2fbf9c" +dependencies = [ + "dupe_derive", +] + +[[package]] +name = "dupe_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e195b4945e88836d826124af44fdcb262ec01ef94d44f14f4fb5103f19892a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "ena" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +dependencies = [ + "log", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -745,6 +1014,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enumflags2" version = "0.7.11" @@ -762,7 +1037,7 @@ checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -771,12 +1046,44 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbfd0e7fc632dec5e6c9396a27bc9f9975b4e039720e1fd3e34021d3ce28c415" +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + [[package]] name = "errno" version = "0.3.11" @@ -787,6 +1094,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "error-code" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" + [[package]] name = "event-listener" version = "5.4.0" @@ -846,6 +1159,23 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix 1.0.5", + "windows-sys 0.59.0", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.10.0" @@ -956,7 +1286,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -989,6 +1319,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1041,13 +1380,29 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1071,6 +1426,27 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "1.3.1" @@ -1330,7 +1706,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1366,6 +1742,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.9.0" @@ -1373,7 +1760,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", + "serde", ] [[package]] @@ -1392,7 +1780,16 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn", + "syn 2.0.100", +] + +[[package]] +name = "inventory" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab08d7cd2c5897f2c949e5383ea7c7db03fb19130ffcfbf7eda795137ae3cb83" +dependencies = [ + "rustversion", ] [[package]] @@ -1401,12 +1798,32 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1422,6 +1839,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a064218214dc6a10fbae5ec5fa888d80c45d611aba169222fc272072bf7aef6" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1432,6 +1873,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lalrpop" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b" +dependencies = [ + "ascii-canvas", + "bit-set", + "diff", + "ena", + "is-terminal", + "itertools 0.10.5", + "lalrpop-util", + "petgraph", + "regex", + "regex-syntax 0.6.29", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", +] + +[[package]] +name = "lalrpop-util" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed" +dependencies = [ + "regex", +] + [[package]] name = "landlock" version = "0.4.1" @@ -1461,7 +1933,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags", + "bitflags 2.9.0", "libc", ] @@ -1499,15 +1971,57 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "logos" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8b031682c67a8e3d5446840f9573eb7fe26efe7ec8d195c9ac4c0647c502f1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 1.0.109", +] + [[package]] name = "lru" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown", + "hashbrown 0.15.2", ] +[[package]] +name = "lsp-types" +version = "0.94.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66bfd44a06ae10647fe3f8214762e9369fd4248df1350924b4ef9e770a85ea1" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.1.0" @@ -1523,6 +2037,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1566,6 +2089,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +dependencies = [ + "serde", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -1583,6 +2115,33 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1620,12 +2179,31 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1641,7 +2219,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -1666,7 +2244,7 @@ version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ - "bitflags", + "bitflags 2.9.0", "cfg-if", "foreign-types", "libc", @@ -1683,7 +2261,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1784,12 +2362,49 @@ dependencies = [ "nom_locate", ] +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.9.0", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1808,6 +2423,21 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1820,9 +2450,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.24", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "predicates" version = "3.1.3" @@ -1897,6 +2533,16 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.9.1" @@ -1932,13 +2578,13 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags", + "bitflags 2.9.0", "cassowary", "compact_str", "crossterm", "indoc", "instability", - "itertools", + "itertools 0.13.0", "lru", "paste", "strum", @@ -1959,7 +2605,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -1973,6 +2619,17 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.0" @@ -1984,6 +2641,26 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "regex" version = "1.11.1" @@ -2112,7 +2789,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2125,7 +2802,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ - "bitflags", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.9.4", @@ -2177,6 +2854,28 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "rustyline" +version = "14.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "clipboard-win", + "fd-lock", + "home", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "unicode-segmentation", + "unicode-width 0.1.14", + "utf8parse", + "windows-sys 0.52.0", +] + [[package]] name = "ryu" version = "1.0.20" @@ -2192,6 +2891,48 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemafy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aea5ba40287dae331f2c48b64dbc8138541f5e97ee8793caa7948c1f31d86d5" +dependencies = [ + "Inflector", + "schemafy_core", + "schemafy_lib", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "syn 1.0.109", +] + +[[package]] +name = "schemafy_core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41781ae092f4fd52c9287efb74456aea0d3b90032d2ecad272bd14dbbcb0511b" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "schemafy_lib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e953db32579999ca98c451d80801b6f6a7ecba6127196c5387ec0774c528befa" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "schemafy_core", + "serde", + "serde_derive", + "serde_json", + "syn 1.0.109", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2213,7 +2954,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.9.0", "core-foundation", "core-foundation-sys", "libc", @@ -2247,7 +2988,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2256,13 +2997,24 @@ version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ - "indexmap", + "indexmap 2.9.0", "itoa", "memchr", "ryu", "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -2284,6 +3036,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.9.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2341,6 +3123,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -2372,6 +3160,96 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "starlark" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f53849859f05d9db705b221bd92eede93877fd426c1b4a3c3061403a5912a8f" +dependencies = [ + "allocative", + "anyhow", + "bumpalo", + "cmp_any", + "debugserver-types", + "derivative", + "derive_more", + "display_container", + "dupe", + "either", + "erased-serde", + "hashbrown 0.14.5", + "inventory", + "itertools 0.13.0", + "maplit", + "memoffset", + "num-bigint", + "num-traits", + "once_cell", + "paste", + "ref-cast", + "regex", + "rustyline", + "serde", + "serde_json", + "starlark_derive", + "starlark_map", + "starlark_syntax", + "static_assertions", + "strsim 0.10.0", + "textwrap", + "thiserror 1.0.69", +] + +[[package]] +name = "starlark_derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe58bc6c8b7980a1fe4c9f8f48200c3212db42ebfe21ae6a0336385ab53f082a" +dependencies = [ + "dupe", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "starlark_map" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92659970f120df0cc1c0bb220b33587b7a9a90e80d4eecc5c5af5debb950173d" +dependencies = [ + "allocative", + "dupe", + "equivalent", + "fxhash", + "hashbrown 0.14.5", + "serde", +] + +[[package]] +name = "starlark_syntax" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe53b3690d776aafd7cb6b9fed62d94f83280e3b87d88e3719cc0024638461b3" +dependencies = [ + "allocative", + "annotate-snippets", + "anyhow", + "derivative", + "derive_more", + "dupe", + "lalrpop", + "lalrpop-util", + "logos", + "lsp-types", + "memchr", + "num-bigint", + "num-traits", + "once_cell", + "starlark_map", + "thiserror 1.0.69", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2384,6 +3262,24 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2409,7 +3305,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.100", ] [[package]] @@ -2418,6 +3314,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.100" @@ -2446,7 +3353,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2455,7 +3362,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags", + "bitflags 2.9.0", "core-foundation", "system-configuration-sys", ] @@ -2483,6 +3390,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "terminal_size" version = "0.4.2" @@ -2499,6 +3417,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2525,7 +3452,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2536,7 +3463,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2580,6 +3507,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2615,7 +3551,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2678,7 +3614,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", @@ -2744,7 +3680,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2877,7 +3813,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ - "itertools", + "itertools 0.13.0", "unicode-segmentation", "unicode-width 0.1.14", ] @@ -2894,6 +3830,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -2909,6 +3851,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2941,6 +3884,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -3002,7 +3951,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -3037,7 +3986,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3117,7 +4066,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3128,7 +4077,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3360,7 +4309,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -3401,17 +4350,37 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive 0.7.35", +] + [[package]] name = "zerocopy" version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.8.24", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -3422,7 +4391,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3442,7 +4411,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] @@ -3471,5 +4440,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index f3f66eb2d7..69c4e8a8a0 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -6,6 +6,7 @@ members = [ "cli", "core", "exec", + "execpolicy", "interactive", "repl", "tui", diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml new file mode 100644 index 0000000000..6d8fd5ac05 --- /dev/null +++ b/codex-rs/execpolicy/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "codex-execpolicy" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "codex-execpolicy" +path = "src/main.rs" + +[lib] +name = "codex_execpolicy" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +starlark = "0.13.0" +allocative = "0.3.3" +clap = { version = "4", features = ["derive"] } +derive_more = { version = "1", features = ["display"] } +env_logger = "0.11.5" +log = "0.4" +multimap = "0.10.0" +path-absolutize = "3.1.1" +regex = "1.11.1" +serde = { version = "1.0.194", features = ["derive"] } +serde_json = "1.0.110" +serde_with = { version = "3", features = ["macros"] } +tempfile = "3.13.0" diff --git a/codex-rs/execpolicy/README.md b/codex-rs/execpolicy/README.md new file mode 100644 index 0000000000..1d783a6001 --- /dev/null +++ b/codex-rs/execpolicy/README.md @@ -0,0 +1,180 @@ +# codex_execpolicy + +The goal of this library is to classify a proposed `execv(3)` command into one of the following states: + +- `safe` The command is safe to run (\*). +- `match` The command matched a rule in the policy, but the caller should decide whether it is safe to run based on the files it will write. +- `forbidden` The command is not allowed to be run. +- `unverified` The safety cannot be determined: make the user decide. + +(\*) Whether an `execv(3)` call should be considered "safe" often requires additional context beyond the arguments to `execv()` itself. For example, if you trust an autonomous software agent to write files in your source tree, then deciding whether `/bin/cp foo bar` is "safe" depends on `getcwd(3)` for the calling process as well as the `realpath` of `foo` and `bar` when resolved against `getcwd()`. +To that end, rather than returning a boolean, the validator returns a structured result that the client is expected to use to determine the "safety" of the proposed `execv()` call. + +For example, to check the command `ls -l foo`, the checker would be invoked as follows: + +```shell +cargo run -- check ls -l foo | jq +``` + +It will exit with `0` and print the following to stdout: + +```json +{ + "result": "safe", + "match": { + "program": "ls", + "flags": [ + { + "name": "-l" + } + ], + "opts": [], + "args": [ + { + "index": 1, + "type": "ReadableFile", + "value": "foo" + } + ], + "system_path": ["/bin/ls", "/usr/bin/ls"] + } +} +``` + +Of note: + +- `foo` is tagged as a `ReadableFile`, so the caller should resolve `foo` relative to `getcwd()` and `realpath` it (as it may be a symlink) to determine whether `foo` is safe to read. +- While the specified executable is `ls`, `"system_path"` offers `/bin/ls` and `/usr/bin/ls` as viable alternatives to avoid using whatever `ls` happens to appear first on the user's `$PATH`. If either exists on the host, it is recommended to use it as the first argument to `execv(3)` instead of `ls`. + +Further, "safety" in this system is not a guarantee that the command will execute successfully. As an example, `cat /Users/mbolin/code/codex/README.md` may be considered "safe" if the system has decided the agent is allowed to read anything under `/Users/mbolin/code/codex`, but it will fail at runtime because `README.md` does not exist. (Though this is "safe" in that the agent did not read any files that it was not authorized to read.) + +## Policy + +Currently, the default policy is defined in [`default.policy`](./src/default.policy) within the crate. + +The system uses [Starlark](https://bazel.build/rules/language) as the file format because, unlike something like JSON or YAML, it supports "macros" without compromising on safety or reproducibility. (Under the hood, we use [`starlark-rust`](https://github.com/facebook/starlark-rust) as the specific Starlark implementation.) + +This policy contains "rules" such as: + +```python +define_program( + program="cp", + options=[ + flag("-r"), + flag("-R"), + flag("--recursive"), + ], + args=[ARG_RFILES, ARG_WFILE], + system_path=["/bin/cp", "/usr/bin/cp"], + should_match=[ + ["foo", "bar"], + ], + should_not_match=[ + ["foo"], + ], +) +``` + +This rule means that: + +- `cp` can be used with any of the following flags (where "flag" means "an option that does not take an argument"): `-r`, `-R`, `--recursive`. +- The initial `ARG_RFILES` passed to `args` means that it expects one or more arguments that correspond to "readable files" +- The final `ARG_WFILE` passed to `args` means that it expects exactly one argument that corresponds to a "writeable file." +- As a means of a lightweight way of including a unit test alongside the definition, the `should_match` list is a list of examples of `execv(3)` args that should match the rule and `should_not_match` is a list of examples that should not match. These examples are verified when the `.policy` file is loaded. + +Note that the language of the `.policy` file is still evolving, as we have to continue to expand it so it is sufficiently expressive to accept all commands we want to consider "safe" without allowing unsafe commands to pass through. + +The integrity of `default.policy` is verified [via unit tests](./tests). + +Further, the CLI supports a `--policy` option to specify a custom `.policy` file for ad-hoc testing. + +## Output Type: `match` + +Going back to the `cp` example, because the rule matches an `ARG_WFILE`, it will return `match` instead of `safe`: + +```shell +cargo run -- check cp src1 src2 dest | jq +``` + +If the caller wants to consider allowing this command, it should parse the JSON to pick out the `WriteableFile` arguments and decide whether they are safe to write: + +```json +{ + "result": "match", + "match": { + "program": "cp", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": "ReadableFile", + "value": "src1" + }, + { + "index": 1, + "type": "ReadableFile", + "value": "src2" + }, + { + "index": 2, + "type": "WriteableFile", + "value": "dest" + } + ], + "system_path": ["/bin/cp", "/usr/bin/cp"] + } +} +``` + +Note the exit code is still `0` for a `match` unless the `--require-safe` flag is specified, in which case the exit code is `12`. + +## Output Type: `forbidden` + +It is also possible to define a rule that, if it matches a command, should flag it as _forbidden_. For example, we do not want agents to be able to run `applied deploy` _ever_, so we define the following rule: + +```python +define_program( + program="applied", + args=["deploy"], + forbidden="Infrastructure Risk: command contains 'applied deploy'", + should_match=[ + ["deploy"], + ], + should_not_match=[ + ["lint"], + ], +) +``` + +Note that for a rule to be forbidden, the `forbidden` keyword arg must be specified as the reason the command is forbidden. This will be included in the output: + +```shell +cargo run -- check applied deploy | jq +``` + +```json +{ + "result": "forbidden", + "reason": "Infrastructure Risk: command contains 'applied deploy'", + "cause": { + "Exec": { + "exec": { + "program": "applied", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": { + "Literal": "deploy" + }, + "value": "deploy" + } + ], + "system_path": [] + } + } + } +} +``` diff --git a/codex-rs/execpolicy/build.rs b/codex-rs/execpolicy/build.rs new file mode 100644 index 0000000000..eda4846853 --- /dev/null +++ b/codex-rs/execpolicy/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=src/default.policy"); +} diff --git a/codex-rs/execpolicy/src/arg_matcher.rs b/codex-rs/execpolicy/src/arg_matcher.rs new file mode 100644 index 0000000000..12d91b4465 --- /dev/null +++ b/codex-rs/execpolicy/src/arg_matcher.rs @@ -0,0 +1,118 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::arg_type::ArgType; +use crate::starlark::values::ValueLike; +use allocative::Allocative; +use derive_more::derive::Display; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::string::StarlarkStr; +use starlark::values::AllocValue; +use starlark::values::Heap; +use starlark::values::NoSerialize; +use starlark::values::StarlarkValue; +use starlark::values::UnpackValue; +use starlark::values::Value; + +/// Patterns that lists of arguments should be compared against. +#[derive(Clone, Debug, Display, Eq, PartialEq, NoSerialize, ProvidesStaticType, Allocative)] +#[display("{}", self)] +pub enum ArgMatcher { + /// Literal string value. + Literal(String), + + /// We cannot say what type of value this should match, but it is *not* a file path. + OpaqueNonFile, + + /// Required readable file. + ReadableFile, + + /// Required writeable file. + WriteableFile, + + /// Non-empty list of readable files. + ReadableFiles, + + /// Non-empty list of readable files, or empty list, implying readable cwd. + ReadableFilesOrCwd, + + /// Positive integer, like one that is required for `head -n`. + PositiveInteger, + + /// Bespoke matcher for safe sed commands. + SedCommand, + + /// Matches an arbitrary number of arguments without attributing any + /// particular meaning to them. Caller is responsible for interpreting them. + UnverifiedVarargs, +} + +impl ArgMatcher { + pub fn cardinality(&self) -> ArgMatcherCardinality { + match self { + ArgMatcher::Literal(_) + | ArgMatcher::OpaqueNonFile + | ArgMatcher::ReadableFile + | ArgMatcher::WriteableFile + | ArgMatcher::PositiveInteger + | ArgMatcher::SedCommand => ArgMatcherCardinality::One, + ArgMatcher::ReadableFiles => ArgMatcherCardinality::AtLeastOne, + ArgMatcher::ReadableFilesOrCwd | ArgMatcher::UnverifiedVarargs => { + ArgMatcherCardinality::ZeroOrMore + } + } + } + + pub fn arg_type(&self) -> ArgType { + match self { + ArgMatcher::Literal(value) => ArgType::Literal(value.clone()), + ArgMatcher::OpaqueNonFile => ArgType::OpaqueNonFile, + ArgMatcher::ReadableFile => ArgType::ReadableFile, + ArgMatcher::WriteableFile => ArgType::WriteableFile, + ArgMatcher::ReadableFiles => ArgType::ReadableFile, + ArgMatcher::ReadableFilesOrCwd => ArgType::ReadableFile, + ArgMatcher::PositiveInteger => ArgType::PositiveInteger, + ArgMatcher::SedCommand => ArgType::SedCommand, + ArgMatcher::UnverifiedVarargs => ArgType::Unknown, + } + } +} + +pub enum ArgMatcherCardinality { + One, + AtLeastOne, + ZeroOrMore, +} + +impl ArgMatcherCardinality { + pub fn is_exact(&self) -> Option { + match self { + ArgMatcherCardinality::One => Some(1), + ArgMatcherCardinality::AtLeastOne => None, + ArgMatcherCardinality::ZeroOrMore => None, + } + } +} + +impl<'v> AllocValue<'v> for ArgMatcher { + fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + heap.alloc_simple(self) + } +} + +#[starlark_value(type = "ArgMatcher")] +impl<'v> StarlarkValue<'v> for ArgMatcher { + type Canonical = ArgMatcher; +} + +impl<'v> UnpackValue<'v> for ArgMatcher { + type Error = starlark::Error; + + fn unpack_value_impl(value: Value<'v>) -> starlark::Result> { + if let Some(str) = value.downcast_ref::() { + Ok(Some(ArgMatcher::Literal(str.as_str().to_string()))) + } else { + Ok(value.downcast_ref::().cloned()) + } + } +} diff --git a/codex-rs/execpolicy/src/arg_resolver.rs b/codex-rs/execpolicy/src/arg_resolver.rs new file mode 100644 index 0000000000..d1138a8ffc --- /dev/null +++ b/codex-rs/execpolicy/src/arg_resolver.rs @@ -0,0 +1,194 @@ +use serde::Serialize; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_matcher::ArgMatcherCardinality; +use crate::error::Error; +use crate::error::Result; +use crate::valid_exec::MatchedArg; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct PositionalArg { + pub index: usize, + pub value: String, +} + +pub fn resolve_observed_args_with_patterns( + program: &str, + args: Vec, + arg_patterns: &Vec, +) -> Result> { + // Naive matching implementation. Among `arg_patterns`, there is allowed to + // be at most one vararg pattern. Assuming `arg_patterns` is non-empty, we + // end up with either: + // + // - all `arg_patterns` in `prefix_patterns` + // - `arg_patterns` split across `prefix_patterns` (which could be empty), + // one `vararg_pattern`, and `suffix_patterns` (which could also empty). + // + // From there, we start by matching everything in `prefix_patterns`. + // Then we calculate how many positional args should be matched by + // `suffix_patterns` and use that to determine how many args are left to + // be matched by `vararg_pattern` (which could be zero). + // + // After assocating positional args with `vararg_pattern`, we match the + // `suffix_patterns` with the remaining args. + let ParitionedArgs { + num_prefix_args, + num_suffix_args, + prefix_patterns, + suffix_patterns, + vararg_pattern, + } = partition_args(program, arg_patterns)?; + + let mut matched_args = Vec::::new(); + + let prefix = get_range_checked(&args, 0..num_prefix_args)?; + let mut prefix_arg_index = 0; + for pattern in prefix_patterns { + let n = pattern.cardinality().is_exact().unwrap(); + for positional_arg in &prefix[prefix_arg_index..prefix_arg_index + n] { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + prefix_arg_index += n; + } + + if num_suffix_args > args.len() { + return Err(Error::NotEnoughArgs { + program: program.to_string(), + args, + arg_patterns: arg_patterns.clone(), + }); + } + + let initial_suffix_args_index = args.len() - num_suffix_args; + if prefix_arg_index > initial_suffix_args_index { + return Err(Error::PrefixOverlapsSuffix {}); + } + + if let Some(pattern) = vararg_pattern { + let vararg = get_range_checked(&args, prefix_arg_index..initial_suffix_args_index)?; + match pattern.cardinality() { + ArgMatcherCardinality::One => { + return Err(Error::InternalInvariantViolation { + message: "vararg pattern should not have cardinality of one".to_string(), + }); + } + ArgMatcherCardinality::AtLeastOne => { + if vararg.is_empty() { + return Err(Error::VarargMatcherDidNotMatchAnything { + program: program.to_string(), + matcher: pattern, + }); + } else { + for positional_arg in vararg { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + } + } + ArgMatcherCardinality::ZeroOrMore => { + for positional_arg in vararg { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + } + } + } + + let suffix = get_range_checked(&args, initial_suffix_args_index..args.len())?; + let mut suffix_arg_index = 0; + for pattern in suffix_patterns { + let n = pattern.cardinality().is_exact().unwrap(); + for positional_arg in &suffix[suffix_arg_index..suffix_arg_index + n] { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + suffix_arg_index += n; + } + + if matched_args.len() < args.len() { + let extra_args = get_range_checked(&args, matched_args.len()..args.len())?; + Err(Error::UnexpectedArguments { + program: program.to_string(), + args: extra_args.to_vec(), + }) + } else { + Ok(matched_args) + } +} + +#[derive(Default)] +struct ParitionedArgs { + num_prefix_args: usize, + num_suffix_args: usize, + prefix_patterns: Vec, + suffix_patterns: Vec, + vararg_pattern: Option, +} + +fn partition_args(program: &str, arg_patterns: &Vec) -> Result { + let mut in_prefix = true; + let mut partitioned_args = ParitionedArgs::default(); + + for pattern in arg_patterns { + match pattern.cardinality().is_exact() { + Some(n) => { + if in_prefix { + partitioned_args.prefix_patterns.push(pattern.clone()); + partitioned_args.num_prefix_args += n; + } else { + partitioned_args.suffix_patterns.push(pattern.clone()); + partitioned_args.num_suffix_args += n; + } + } + None => match partitioned_args.vararg_pattern { + None => { + partitioned_args.vararg_pattern = Some(pattern.clone()); + in_prefix = false; + } + Some(existing_pattern) => { + return Err(Error::MultipleVarargPatterns { + program: program.to_string(), + first: existing_pattern, + second: pattern.clone(), + }); + } + }, + } + } + + Ok(partitioned_args) +} + +fn get_range_checked(vec: &[T], range: std::ops::Range) -> Result<&[T]> { + if range.start > range.end { + Err(Error::RangeStartExceedsEnd { + start: range.start, + end: range.end, + }) + } else if range.end > vec.len() { + Err(Error::RangeEndOutOfBounds { + end: range.end, + len: vec.len(), + }) + } else { + Ok(&vec[range]) + } +} diff --git a/codex-rs/execpolicy/src/arg_type.rs b/codex-rs/execpolicy/src/arg_type.rs new file mode 100644 index 0000000000..11be0277ec --- /dev/null +++ b/codex-rs/execpolicy/src/arg_type.rs @@ -0,0 +1,87 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::error::Error; +use crate::error::Result; +use crate::sed_command::parse_sed_command; +use allocative::Allocative; +use derive_more::derive::Display; +use serde::Serialize; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::StarlarkValue; + +#[derive(Debug, Clone, Display, Eq, PartialEq, ProvidesStaticType, Allocative, Serialize)] +#[display("{}", self)] +pub enum ArgType { + Literal(String), + /// We cannot say what this argument represents, but it is *not* a file path. + OpaqueNonFile, + /// A file (or directory) that can be expected to be read as part of this command. + ReadableFile, + /// A file (or directory) that can be expected to be written as part of this command. + WriteableFile, + /// Positive integer, like one that is required for `head -n`. + PositiveInteger, + /// Bespoke arg type for a safe sed command. + SedCommand, + /// Type is unknown: it may or may not be a file. + Unknown, +} + +impl ArgType { + pub fn validate(&self, value: &str) -> Result<()> { + match self { + ArgType::Literal(literal_value) => { + if value != *literal_value { + Err(Error::LiteralValueDidNotMatch { + expected: literal_value.clone(), + actual: value.to_string(), + }) + } else { + Ok(()) + } + } + ArgType::ReadableFile => { + if value.is_empty() { + Err(Error::EmptyFileName {}) + } else { + Ok(()) + } + } + ArgType::WriteableFile => { + if value.is_empty() { + Err(Error::EmptyFileName {}) + } else { + Ok(()) + } + } + ArgType::OpaqueNonFile | ArgType::Unknown => Ok(()), + ArgType::PositiveInteger => match value.parse::() { + Ok(0) => Err(Error::InvalidPositiveInteger { + value: value.to_string(), + }), + Ok(_) => Ok(()), + Err(_) => Err(Error::InvalidPositiveInteger { + value: value.to_string(), + }), + }, + ArgType::SedCommand => parse_sed_command(value), + } + } + + pub fn might_write_file(&self) -> bool { + match self { + ArgType::WriteableFile | ArgType::Unknown => true, + ArgType::Literal(_) + | ArgType::OpaqueNonFile + | ArgType::PositiveInteger + | ArgType::ReadableFile + | ArgType::SedCommand => false, + } + } +} + +#[starlark_value(type = "ArgType")] +impl<'v> StarlarkValue<'v> for ArgType { + type Canonical = ArgType; +} diff --git a/codex-rs/execpolicy/src/default.policy b/codex-rs/execpolicy/src/default.policy new file mode 100644 index 0000000000..bd27a0bb30 --- /dev/null +++ b/codex-rs/execpolicy/src/default.policy @@ -0,0 +1,202 @@ +""" +define_program() supports the following arguments: +- program: the name of the program +- system_path: list of absolute paths on the system where program can likely be found +- option_bundling (PLANNED): whether to allow bundling of options (e.g. `-al` for `-a -l`) +- combine_format (PLANNED): whether to allow `--option=value` (as opposed to `--option value`) +- options: the command-line flags/options: use flag() and opt() to define these +- args: the rules for what arguments are allowed that are not "options" +- should_match: list of command-line invocations that should be matched by the rule +- should_not_match: list of command-line invocations that should not be matched by the rule +""" + +define_program( + program="ls", + system_path=["/bin/ls", "/usr/bin/ls"], + options=[ + flag("-1"), + flag("-a"), + flag("-l"), + ], + args=[ARG_RFILES_OR_CWD], +) + +define_program( + program="cat", + options=[ + flag("-b"), + flag("-n"), + flag("-t"), + ], + system_path=["/bin/cat", "/usr/bin/cat"], + args=[ARG_RFILES], + should_match=[ + ["file.txt"], + ["-n", "file.txt"], + ["-b", "file.txt"], + ], + should_not_match=[ + # While cat without args is valid, it will read from stdin, which + # does not seem appropriate for our current use case. + [], + # Let's not auto-approve advisory locking. + ["-l", "file.txt"], + ] +) + +define_program( + program="cp", + options=[ + flag("-r"), + flag("-R"), + flag("--recursive"), + ], + args=[ARG_RFILES, ARG_WFILE], + system_path=["/bin/cp", "/usr/bin/cp"], + should_match=[ + ["foo", "bar"], + ], + should_not_match=[ + ["foo"], + ], +) + +define_program( + program="head", + system_path=["/bin/head", "/usr/bin/head"], + options=[ + opt("-c", ARG_POS_INT), + opt("-n", ARG_POS_INT), + ], + args=[ARG_RFILES], +) + +printenv_system_path = ["/usr/bin/printenv"] + +# Print all environment variables. +define_program( + program="printenv", + args=[], + system_path=printenv_system_path, + # This variant of `printenv` only allows zero args. + should_match=[[]], + should_not_match=[["PATH"]], +) + +# Print a specific environment variable. +define_program( + program="printenv", + args=[ARG_OPAQUE_VALUE], + system_path=printenv_system_path, + # This variant of `printenv` only allows exactly one arg. + should_match=[["PATH"]], + should_not_match=[[], ["PATH", "HOME"]], +) + +# Note that `pwd` is generally implemented as a shell built-in. It does not +# accept any arguments. +define_program( + program="pwd", + options=[ + flag("-L"), + flag("-P"), + ], + args=[], +) + +define_program( + program="rg", + options=[ + opt("-A", ARG_POS_INT), + opt("-B", ARG_POS_INT), + opt("-C", ARG_POS_INT), + opt("-d", ARG_POS_INT), + opt("--max-depth", ARG_POS_INT), + opt("-g", ARG_OPAQUE_VALUE), + opt("--glob", ARG_OPAQUE_VALUE), + opt("-m", ARG_POS_INT), + opt("--max-count", ARG_POS_INT), + + flag("-n"), + flag("-i"), + flag("-l"), + flag("--files"), + flag("--files-with-matches"), + flag("--files-without-match"), + ], + args=[ARG_OPAQUE_VALUE, ARG_RFILES_OR_CWD], + should_match=[ + ["-n", "init"], + ["-n", "init", "."], + ["-i", "-n", "init", "src"], + ["--files", "--max-depth", "2", "."], + ], + should_not_match=[ + ["-m", "-n", "init"], + ["--glob", "src"], + ], + # TODO(mbolin): Perhaps we need a way to indicate that we expect `rg` to be + # bundled with the host environment and we should be using that verison. + system_path=[], +) + +# Unfortunately, `sed` is difficult to secure because GNU sed supports an `e` +# flag where `s/pattern/replacement/e` would run `replacement` as a shell +# command every time `pattern` is matched. For example, try the following on +# Ubuntu (which uses GNU sed, unlike macOS): +# +# ```shell +# $ yes | head -n 4 > /tmp/yes.txt +# $ sed 's/y/echo hi/e' /tmp/yes.txt +# hi +# hi +# hi +# hi +# ``` +# +# As you can see, `echo hi` got executed four times. In order to support some +# basic sed functionality, we implement a bespoke `ARG_SED_COMMAND` that matches +# only "known safe" sed commands. +common_sed_flags = [ + # We deliberately do not support -i or -f. + flag("-n"), + flag("-u"), +] +sed_system_path = ["/usr/bin/sed"] + +# When -e is not specified, the first argument must be a valid sed command. +define_program( + program="sed", + options=common_sed_flags, + args=[ARG_SED_COMMAND, ARG_RFILES], + system_path=sed_system_path, +) + +# When -e is required, all arguments are assumed to be readable files. +define_program( + program="sed", + options=common_sed_flags + [ + opt("-e", ARG_SED_COMMAND, required=True), + ], + args=[ARG_RFILES], + system_path=sed_system_path, +) + +define_program( + program="which", + options=[ + flag("-a"), + flag("-s"), + ], + # Surprisingly, `which` takes more than one argument. + args=[ARG_RFILES], + should_match=[ + ["python3"], + ["-a", "python3"], + ["-a", "python3", "cargo"], + ], + should_not_match=[ + [], + ], + system_path=["/bin/which", "/usr/bin/which"], +) diff --git a/codex-rs/execpolicy/src/error.rs b/codex-rs/execpolicy/src/error.rs new file mode 100644 index 0000000000..ff781f43a5 --- /dev/null +++ b/codex-rs/execpolicy/src/error.rs @@ -0,0 +1,96 @@ +use std::path::PathBuf; + +use serde::Serialize; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_resolver::PositionalArg; +use serde_with::serde_as; +use serde_with::DisplayFromStr; + +pub type Result = std::result::Result; + +#[serde_as] +#[derive(Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "type")] +pub enum Error { + NoSpecForProgram { + program: String, + }, + OptionMissingValue { + program: String, + option: String, + }, + OptionFollowedByOptionInsteadOfValue { + program: String, + option: String, + value: String, + }, + UnknownOption { + program: String, + option: String, + }, + UnexpectedArguments { + program: String, + args: Vec, + }, + DoubleDashNotSupportedYet { + program: String, + }, + MultipleVarargPatterns { + program: String, + first: ArgMatcher, + second: ArgMatcher, + }, + RangeStartExceedsEnd { + start: usize, + end: usize, + }, + RangeEndOutOfBounds { + end: usize, + len: usize, + }, + PrefixOverlapsSuffix {}, + NotEnoughArgs { + program: String, + args: Vec, + arg_patterns: Vec, + }, + InternalInvariantViolation { + message: String, + }, + VarargMatcherDidNotMatchAnything { + program: String, + matcher: ArgMatcher, + }, + EmptyFileName {}, + LiteralValueDidNotMatch { + expected: String, + actual: String, + }, + InvalidPositiveInteger { + value: String, + }, + MissingRequiredOptions { + program: String, + options: Vec, + }, + SedCommandNotProvablySafe { + command: String, + }, + ReadablePathNotInReadableFolders { + file: PathBuf, + folders: Vec, + }, + WriteablePathNotInWriteableFolders { + file: PathBuf, + folders: Vec, + }, + CannotCheckRelativePath { + file: PathBuf, + }, + CannotCanonicalizePath { + file: String, + #[serde_as(as = "DisplayFromStr")] + error: std::io::ErrorKind, + }, +} diff --git a/codex-rs/execpolicy/src/exec_call.rs b/codex-rs/execpolicy/src/exec_call.rs new file mode 100644 index 0000000000..e9753eccf3 --- /dev/null +++ b/codex-rs/execpolicy/src/exec_call.rs @@ -0,0 +1,28 @@ +use std::fmt::Display; + +use serde::Serialize; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ExecCall { + pub program: String, + pub args: Vec, +} + +impl ExecCall { + pub fn new(program: &str, args: &[&str]) -> Self { + Self { + program: program.to_string(), + args: args.iter().map(|&s| s.into()).collect(), + } + } +} + +impl Display for ExecCall { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.program)?; + for arg in &self.args { + write!(f, " {}", arg)?; + } + Ok(()) + } +} diff --git a/codex-rs/execpolicy/src/execv_checker.rs b/codex-rs/execpolicy/src/execv_checker.rs new file mode 100644 index 0000000000..787fbce122 --- /dev/null +++ b/codex-rs/execpolicy/src/execv_checker.rs @@ -0,0 +1,263 @@ +use std::ffi::OsString; +use std::path::Path; +use std::path::PathBuf; + +use crate::ArgType; +use crate::Error::CannotCanonicalizePath; +use crate::Error::CannotCheckRelativePath; +use crate::Error::ReadablePathNotInReadableFolders; +use crate::Error::WriteablePathNotInWriteableFolders; +use crate::ExecCall; +use crate::MatchedExec; +use crate::Policy; +use crate::Result; +use crate::ValidExec; +use path_absolutize::*; +use std::os::unix::fs::PermissionsExt; + +macro_rules! check_file_in_folders { + ($file:expr, $folders:expr, $error:ident) => { + if !$folders.iter().any(|folder| $file.starts_with(folder)) { + return Err($error { + file: $file.clone(), + folders: $folders.to_vec(), + }); + } + }; +} + +pub struct ExecvChecker { + execv_policy: Policy, +} + +impl ExecvChecker { + pub fn new(execv_policy: Policy) -> Self { + Self { execv_policy } + } + + pub fn r#match(&self, exec_call: &ExecCall) -> Result { + self.execv_policy.check(exec_call) + } + + /// The caller is responsible for ensuring readable_folders and + /// writeable_folders are in canonical form. + pub fn check( + &self, + valid_exec: ValidExec, + cwd: &Option, + readable_folders: &[PathBuf], + writeable_folders: &[PathBuf], + ) -> Result { + for (arg_type, value) in valid_exec + .args + .into_iter() + .map(|arg| (arg.r#type, arg.value)) + .chain( + valid_exec + .opts + .into_iter() + .map(|opt| (opt.r#type, opt.value)), + ) + { + match arg_type { + ArgType::ReadableFile => { + let readable_file = ensure_absolute_path(&value, cwd)?; + check_file_in_folders!( + readable_file, + readable_folders, + ReadablePathNotInReadableFolders + ); + } + ArgType::WriteableFile => { + let writeable_file = ensure_absolute_path(&value, cwd)?; + check_file_in_folders!( + writeable_file, + writeable_folders, + WriteablePathNotInWriteableFolders + ); + } + ArgType::OpaqueNonFile + | ArgType::Unknown + | ArgType::PositiveInteger + | ArgType::SedCommand + | ArgType::Literal(_) => { + continue; + } + } + } + + let mut program = valid_exec.program.to_string(); + for system_path in valid_exec.system_path { + if is_executable_file(&system_path) { + program = system_path.to_string(); + break; + } + } + + Ok(program) + } +} + +fn ensure_absolute_path(path: &str, cwd: &Option) -> Result { + let file = PathBuf::from(path); + let result = if file.is_relative() { + match cwd { + Some(cwd) => file.absolutize_from(cwd), + None => return Err(CannotCheckRelativePath { file }), + } + } else { + file.absolutize() + }; + result + .map(|path| path.into_owned()) + .map_err(|error| CannotCanonicalizePath { + file: path.to_string(), + error: error.kind(), + }) +} + +fn is_executable_file(path: &str) -> bool { + let file_path = Path::new(path); + + if let Ok(metadata) = std::fs::metadata(file_path) { + let permissions = metadata.permissions(); + // Check if the file is executable (by checking the executable bit for the owner) + return metadata.is_file() && (permissions.mode() & 0o111 != 0); + } + + false +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + use crate::MatchedArg; + use crate::PolicyParser; + + fn setup(fake_cp: &Path) -> ExecvChecker { + let source = format!( + r#" +define_program( +program="cp", +args=[ARG_RFILE, ARG_WFILE], +system_path=[{fake_cp:?}] +) +"# + ); + let parser = PolicyParser::new("#test", &source); + let policy = parser.parse().unwrap(); + ExecvChecker::new(policy) + } + + #[test] + fn test_check_valid_input_files() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + + // Create an executable file that can be used with the system_path arg. + let fake_cp = temp_dir.path().join("cp"); + let fake_cp_file = std::fs::File::create(&fake_cp).unwrap(); + let mut permissions = fake_cp_file.metadata().unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&fake_cp, permissions).unwrap(); + + // Create root_path and reference to files under the root. + let root_path = temp_dir.path().to_path_buf(); + let source_path = root_path.join("source"); + let dest_path = root_path.join("dest"); + + let cp = fake_cp.to_str().unwrap().to_string(); + let root = root_path.to_str().unwrap().to_string(); + let source = source_path.to_str().unwrap().to_string(); + let dest = dest_path.to_str().unwrap().to_string(); + + let cwd = Some(root_path.clone().into()); + + let checker = setup(&fake_cp); + let exec_call = ExecCall { + program: "cp".into(), + args: vec![source.clone(), dest.clone()], + }; + let valid_exec = match checker.r#match(&exec_call)? { + MatchedExec::Match { exec } => exec, + unexpected => panic!("Expected a safe exec but got {unexpected:?}"), + }; + + // No readable or writeable folders specified. + assert_eq!( + checker.check(valid_exec.clone(), &cwd, &[], &[]), + Err(ReadablePathNotInReadableFolders { + file: source_path.clone(), + folders: vec![] + }), + ); + + // Only readable folders specified. + assert_eq!( + checker.check(valid_exec.clone(), &cwd, &[root_path.clone()], &[]), + Err(WriteablePathNotInWriteableFolders { + file: dest_path.clone(), + folders: vec![] + }), + ); + + // Both readable and writeable folders specified. + assert_eq!( + checker.check( + valid_exec.clone(), + &cwd, + &[root_path.clone()], + &[root_path.clone()] + ), + Ok(cp.clone()), + ); + + // Args are the readable and writeable folders, not files within the + // folders. + let exec_call_folders_as_args = ExecCall { + program: "cp".into(), + args: vec![root.clone(), root.clone()], + }; + let valid_exec_call_folders_as_args = match checker.r#match(&exec_call_folders_as_args)? { + MatchedExec::Match { exec } => exec, + _ => panic!("Expected a safe exec"), + }; + assert_eq!( + checker.check( + valid_exec_call_folders_as_args, + &cwd, + &[root_path.clone()], + &[root_path.clone()] + ), + Ok(cp.clone()), + ); + + // Specify a parent of a readable folder as input. + let exec_with_parent_of_readable_folder = ValidExec { + program: "cp".into(), + args: vec![ + MatchedArg::new( + 0, + ArgType::ReadableFile, + root_path.parent().unwrap().to_str().unwrap(), + )?, + MatchedArg::new(1, ArgType::WriteableFile, &dest)?, + ], + ..Default::default() + }; + assert_eq!( + checker.check( + exec_with_parent_of_readable_folder, + &cwd, + &[root_path.clone()], + &[dest_path.clone()] + ), + Err(ReadablePathNotInReadableFolders { + file: root_path.parent().unwrap().to_path_buf(), + folders: vec![root_path.clone()] + }), + ); + Ok(()) + } +} diff --git a/codex-rs/execpolicy/src/lib.rs b/codex-rs/execpolicy/src/lib.rs new file mode 100644 index 0000000000..6f12225981 --- /dev/null +++ b/codex-rs/execpolicy/src/lib.rs @@ -0,0 +1,45 @@ +#![allow(clippy::type_complexity)] +#![allow(clippy::too_many_arguments)] +#[macro_use] +extern crate starlark; + +mod arg_matcher; +mod arg_resolver; +mod arg_type; +mod error; +mod exec_call; +mod execv_checker; +mod opt; +mod policy; +mod policy_parser; +mod program; +mod sed_command; +mod valid_exec; + +pub use arg_matcher::ArgMatcher; +pub use arg_resolver::PositionalArg; +pub use arg_type::ArgType; +pub use error::Error; +pub use error::Result; +pub use exec_call::ExecCall; +pub use execv_checker::ExecvChecker; +pub use opt::Opt; +pub use policy::Policy; +pub use policy_parser::PolicyParser; +pub use program::Forbidden; +pub use program::MatchedExec; +pub use program::NegativeExamplePassedCheck; +pub use program::PositiveExampleFailedCheck; +pub use program::ProgramSpec; +pub use sed_command::parse_sed_command; +pub use valid_exec::MatchedArg; +pub use valid_exec::MatchedFlag; +pub use valid_exec::MatchedOpt; +pub use valid_exec::ValidExec; + +const DEFAULT_POLICY: &str = include_str!("default.policy"); + +pub fn get_default_policy() -> starlark::Result { + let parser = PolicyParser::new("#default", DEFAULT_POLICY); + parser.parse() +} diff --git a/codex-rs/execpolicy/src/main.rs b/codex-rs/execpolicy/src/main.rs new file mode 100644 index 0000000000..d8cb034d2a --- /dev/null +++ b/codex-rs/execpolicy/src/main.rs @@ -0,0 +1,166 @@ +use anyhow::Result; +use clap::Parser; +use clap::Subcommand; +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::Policy; +use codex_execpolicy::PolicyParser; +use codex_execpolicy::ValidExec; +use serde::de; +use serde::Deserialize; +use serde::Serialize; +use std::path::PathBuf; +use std::str::FromStr; + +const MATCHED_BUT_WRITES_FILES_EXIT_CODE: i32 = 12; +const MIGHT_BE_SAFE_EXIT_CODE: i32 = 13; +const FORBIDDEN_EXIT_CODE: i32 = 14; + +#[derive(Parser, Deserialize, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + /// If the command fails the policy, exit with 13, but print parseable JSON + /// to stdout. + #[clap(long)] + pub require_safe: bool, + + /// Path to the policy file. + #[clap(long, short = 'p')] + pub policy: Option, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Clone, Debug, Deserialize, Subcommand)] +pub enum Command { + /// Checks the command as if the arguments were the inputs to execv(3). + Check { + #[arg(trailing_var_arg = true)] + command: Vec, + }, + + /// Checks the command encoded as a JSON object. + #[clap(name = "check-json")] + CheckJson { + /// JSON object with "program" (str) and "args" (list[str]) fields. + #[serde(deserialize_with = "deserialize_from_json")] + exec: ExecArg, + }, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ExecArg { + pub program: String, + + #[serde(default)] + pub args: Vec, +} + +fn main() -> Result<()> { + env_logger::init(); + + let args = Args::parse(); + let policy = match args.policy { + Some(policy) => { + let policy_source = policy.to_string_lossy().to_string(); + let unparsed_policy = std::fs::read_to_string(policy)?; + let parser = PolicyParser::new(&policy_source, &unparsed_policy); + parser.parse() + } + None => get_default_policy(), + }; + let policy = policy.map_err(|err| err.into_anyhow())?; + + let exec = match args.command { + Command::Check { command } => match command.split_first() { + Some((first, rest)) => ExecArg { + program: first.to_string(), + args: rest.iter().map(|s| s.to_string()).collect(), + }, + None => { + eprintln!("no command provided"); + std::process::exit(1); + } + }, + Command::CheckJson { exec } => exec, + }; + + let (output, exit_code) = check_command(&policy, exec, args.require_safe); + let json = serde_json::to_string(&output)?; + println!("{}", json); + std::process::exit(exit_code); +} + +fn check_command( + policy: &Policy, + ExecArg { program, args }: ExecArg, + check: bool, +) -> (Output, i32) { + let exec_call = ExecCall { program, args }; + match policy.check(&exec_call) { + Ok(MatchedExec::Match { exec }) => { + if exec.might_write_files() { + let exit_code = if check { + MATCHED_BUT_WRITES_FILES_EXIT_CODE + } else { + 0 + }; + (Output::Match { r#match: exec }, exit_code) + } else { + (Output::Safe { r#match: exec }, 0) + } + } + Ok(MatchedExec::Forbidden { reason, cause }) => { + let exit_code = if check { FORBIDDEN_EXIT_CODE } else { 0 }; + (Output::Forbidden { reason, cause }, exit_code) + } + Err(err) => { + let exit_code = if check { MIGHT_BE_SAFE_EXIT_CODE } else { 0 }; + (Output::Unverified { error: err }, exit_code) + } + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "result")] +pub enum Output { + /// The command is verified as safe. + #[serde(rename = "safe")] + Safe { r#match: ValidExec }, + + /// The command has matched a rule in the policy, but the caller should + /// decide whether it is "safe" given the files it wants to write. + #[serde(rename = "match")] + Match { r#match: ValidExec }, + + /// The user is forbidden from running the command. + #[serde(rename = "forbidden")] + Forbidden { + reason: String, + cause: codex_execpolicy::Forbidden, + }, + + /// The safety of the command could not be verified. + #[serde(rename = "unverified")] + Unverified { error: codex_execpolicy::Error }, +} + +fn deserialize_from_json<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let decoded = serde_json::from_str(&s) + .map_err(|e| serde::de::Error::custom(format!("JSON parse error: {e}")))?; + Ok(decoded) +} + +impl FromStr for ExecArg { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + serde_json::from_str(s).map_err(|e| e.into()) + } +} diff --git a/codex-rs/execpolicy/src/opt.rs b/codex-rs/execpolicy/src/opt.rs new file mode 100644 index 0000000000..4a58037462 --- /dev/null +++ b/codex-rs/execpolicy/src/opt.rs @@ -0,0 +1,77 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::starlark::values::ValueLike; +use crate::ArgType; +use allocative::Allocative; +use derive_more::derive::Display; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::AllocValue; +use starlark::values::Heap; +use starlark::values::NoSerialize; +use starlark::values::StarlarkValue; +use starlark::values::UnpackValue; +use starlark::values::Value; + +/// Command line option that takes a value. +#[derive(Clone, Debug, Display, PartialEq, Eq, ProvidesStaticType, NoSerialize, Allocative)] +#[display("opt({})", opt)] +pub struct Opt { + /// The option as typed on the command line, e.g., `-h` or `--help`. If + /// it can be used in the `--name=value` format, then this should be + /// `--name` (though this is subject to change). + pub opt: String, + pub meta: OptMeta, + pub required: bool, +} + +/// When defining an Opt, use as specific an OptMeta as possible. +#[derive(Clone, Debug, Display, PartialEq, Eq, ProvidesStaticType, NoSerialize, Allocative)] +#[display("{}", self)] +pub enum OptMeta { + /// Option does not take a value. + Flag, + + /// Option takes a single value matching the specified type. + Value(ArgType), +} + +impl Opt { + pub fn new(opt: String, meta: OptMeta, required: bool) -> Self { + Self { + opt, + meta, + required, + } + } + + pub fn name(&self) -> &str { + &self.opt + } +} + +#[starlark_value(type = "Opt")] +impl<'v> StarlarkValue<'v> for Opt { + type Canonical = Opt; +} + +impl<'v> UnpackValue<'v> for Opt { + type Error = starlark::Error; + + fn unpack_value_impl(value: Value<'v>) -> starlark::Result> { + // TODO(mbolin): It fels like this should be doable without cloning? + // Cannot simply consume the value? + Ok(value.downcast_ref::().cloned()) + } +} + +impl<'v> AllocValue<'v> for Opt { + fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + heap.alloc_simple(self) + } +} + +#[starlark_value(type = "OptMeta")] +impl<'v> StarlarkValue<'v> for OptMeta { + type Canonical = OptMeta; +} diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs new file mode 100644 index 0000000000..5ce7d7b917 --- /dev/null +++ b/codex-rs/execpolicy/src/policy.rs @@ -0,0 +1,103 @@ +use multimap::MultiMap; +use regex::Error as RegexError; +use regex::Regex; + +use crate::error::Error; +use crate::error::Result; +use crate::policy_parser::ForbiddenProgramRegex; +use crate::program::PositiveExampleFailedCheck; +use crate::ExecCall; +use crate::Forbidden; +use crate::MatchedExec; +use crate::NegativeExamplePassedCheck; +use crate::ProgramSpec; + +pub struct Policy { + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings_pattern: Option, +} + +impl Policy { + pub fn new( + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings: Vec, + ) -> std::result::Result { + let forbidden_substrings_pattern = if forbidden_substrings.is_empty() { + None + } else { + let escaped_substrings = forbidden_substrings + .iter() + .map(|s| regex::escape(s)) + .collect::>() + .join("|"); + Some(Regex::new(&format!("({escaped_substrings})"))?) + }; + Ok(Self { + programs, + forbidden_program_regexes, + forbidden_substrings_pattern, + }) + } + + pub fn check(&self, exec_call: &ExecCall) -> Result { + let ExecCall { program, args } = &exec_call; + for ForbiddenProgramRegex { regex, reason } in &self.forbidden_program_regexes { + if regex.is_match(program) { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Program { + program: program.clone(), + exec_call: exec_call.clone(), + }, + reason: reason.clone(), + }); + } + } + + for arg in args { + if let Some(regex) = &self.forbidden_substrings_pattern { + if regex.is_match(arg) { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Arg { + arg: arg.clone(), + exec_call: exec_call.clone(), + }, + reason: format!("arg `{}` contains forbidden substring", arg), + }); + } + } + } + + let mut last_err = Err(Error::NoSpecForProgram { + program: program.clone(), + }); + if let Some(spec_list) = self.programs.get_vec(program) { + for spec in spec_list { + match spec.check(exec_call) { + Ok(matched_exec) => return Ok(matched_exec), + Err(err) => { + last_err = Err(err); + } + } + } + } + last_err + } + + pub fn check_each_good_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_match_list()); + } + violations + } + + pub fn check_each_bad_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_not_match_list()); + } + violations + } +} diff --git a/codex-rs/execpolicy/src/policy_parser.rs b/codex-rs/execpolicy/src/policy_parser.rs new file mode 100644 index 0000000000..caf4efd10d --- /dev/null +++ b/codex-rs/execpolicy/src/policy_parser.rs @@ -0,0 +1,222 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::arg_matcher::ArgMatcher; +use crate::opt::OptMeta; +use crate::Opt; +use crate::Policy; +use crate::ProgramSpec; +use log::info; +use multimap::MultiMap; +use regex::Regex; +use starlark::any::ProvidesStaticType; +use starlark::environment::GlobalsBuilder; +use starlark::environment::LibraryExtension; +use starlark::environment::Module; +use starlark::eval::Evaluator; +use starlark::syntax::AstModule; +use starlark::syntax::Dialect; +use starlark::values::list::UnpackList; +use starlark::values::none::NoneType; +use starlark::values::Heap; +use std::cell::RefCell; +use std::collections::HashMap; + +pub struct PolicyParser { + policy_source: String, + unparsed_policy: String, +} + +impl PolicyParser { + pub fn new(policy_source: &str, unparsed_policy: &str) -> Self { + Self { + policy_source: policy_source.to_string(), + unparsed_policy: unparsed_policy.to_string(), + } + } + + pub fn parse(&self) -> starlark::Result { + let mut dialect = Dialect::Extended.clone(); + dialect.enable_f_strings = true; + let ast = AstModule::parse(&self.policy_source, self.unparsed_policy.clone(), &dialect)?; + let globals = GlobalsBuilder::extended_by(&[LibraryExtension::Typing]) + .with(policy_builtins) + .build(); + let module = Module::new(); + + let heap = Heap::new(); + + module.set("ARG_OPAQUE_VALUE", heap.alloc(ArgMatcher::OpaqueNonFile)); + module.set("ARG_RFILE", heap.alloc(ArgMatcher::ReadableFile)); + module.set("ARG_WFILE", heap.alloc(ArgMatcher::WriteableFile)); + module.set("ARG_RFILES", heap.alloc(ArgMatcher::ReadableFiles)); + module.set( + "ARG_RFILES_OR_CWD", + heap.alloc(ArgMatcher::ReadableFilesOrCwd), + ); + module.set("ARG_POS_INT", heap.alloc(ArgMatcher::PositiveInteger)); + module.set("ARG_SED_COMMAND", heap.alloc(ArgMatcher::SedCommand)); + module.set( + "ARG_UNVERIFIED_VARARGS", + heap.alloc(ArgMatcher::UnverifiedVarargs), + ); + + let policy_builder = PolicyBuilder::new(); + { + let mut eval = Evaluator::new(&module); + eval.extra = Some(&policy_builder); + eval.eval_module(ast, &globals)?; + } + let policy = policy_builder.build(); + policy.map_err(|e| starlark::Error::new_kind(starlark::ErrorKind::Other(e.into()))) + } +} + +#[derive(Debug)] +pub struct ForbiddenProgramRegex { + pub regex: regex::Regex, + pub reason: String, +} + +#[derive(Debug, ProvidesStaticType)] +struct PolicyBuilder { + programs: RefCell>, + forbidden_program_regexes: RefCell>, + forbidden_substrings: RefCell>, +} + +impl PolicyBuilder { + fn new() -> Self { + Self { + programs: RefCell::new(MultiMap::new()), + forbidden_program_regexes: RefCell::new(Vec::new()), + forbidden_substrings: RefCell::new(Vec::new()), + } + } + + fn build(self) -> Result { + let programs = self.programs.into_inner(); + let forbidden_program_regexes = self.forbidden_program_regexes.into_inner(); + let forbidden_substrings = self.forbidden_substrings.into_inner(); + Policy::new(programs, forbidden_program_regexes, forbidden_substrings) + } + + fn add_program_spec(&self, program_spec: ProgramSpec) { + info!("adding program spec: {:?}", program_spec); + let name = program_spec.program.clone(); + let mut programs = self.programs.borrow_mut(); + programs.insert(name.clone(), program_spec); + } + + fn add_forbidden_substrings(&self, substrings: &[String]) { + let mut forbidden_substrings = self.forbidden_substrings.borrow_mut(); + forbidden_substrings.extend_from_slice(substrings); + } + + fn add_forbidden_program_regex(&self, regex: Regex, reason: String) { + let mut forbidden_program_regexes = self.forbidden_program_regexes.borrow_mut(); + forbidden_program_regexes.push(ForbiddenProgramRegex { regex, reason }); + } +} + +#[starlark_module] +fn policy_builtins(builder: &mut GlobalsBuilder) { + fn define_program<'v>( + program: String, + system_path: Option>, + option_bundling: Option, + combined_format: Option, + options: Option>, + args: Option>, + forbidden: Option, + should_match: Option>>, + should_not_match: Option>>, + eval: &mut Evaluator, + ) -> anyhow::Result { + let option_bundling = option_bundling.unwrap_or(false); + let system_path = system_path.map_or_else(Vec::new, |v| v.items.to_vec()); + let combined_format = combined_format.unwrap_or(false); + let options = options.map_or_else(Vec::new, |v| v.items.to_vec()); + let args = args.map_or_else(Vec::new, |v| v.items.to_vec()); + + let mut allowed_options = HashMap::::new(); + for opt in options { + let name = opt.name().to_string(); + if allowed_options + .insert(opt.name().to_string(), opt) + .is_some() + { + return Err(anyhow::format_err!("duplicate flag: {name}")); + } + } + + let program_spec = ProgramSpec::new( + program, + system_path, + option_bundling, + combined_format, + allowed_options, + args, + forbidden, + should_match + .map_or_else(Vec::new, |v| v.items.to_vec()) + .into_iter() + .map(|v| v.items.to_vec()) + .collect(), + should_not_match + .map_or_else(Vec::new, |v| v.items.to_vec()) + .into_iter() + .map(|v| v.items.to_vec()) + .collect(), + ); + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + policy_builder.add_program_spec(program_spec); + Ok(NoneType) + } + + fn forbid_substrings( + strings: UnpackList, + eval: &mut Evaluator, + ) -> anyhow::Result { + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + policy_builder.add_forbidden_substrings(&strings.items.to_vec()); + Ok(NoneType) + } + + fn forbid_program_regex( + regex: String, + reason: String, + eval: &mut Evaluator, + ) -> anyhow::Result { + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + let compiled_regex = regex::Regex::new(®ex)?; + policy_builder.add_forbidden_program_regex(compiled_regex, reason); + Ok(NoneType) + } + + fn opt(name: String, r#type: ArgMatcher, required: Option) -> anyhow::Result { + Ok(Opt::new( + name, + OptMeta::Value(r#type.arg_type()), + required.unwrap_or(false), + )) + } + + fn flag(name: String) -> anyhow::Result { + Ok(Opt::new(name, OptMeta::Flag, false)) + } +} diff --git a/codex-rs/execpolicy/src/program.rs b/codex-rs/execpolicy/src/program.rs new file mode 100644 index 0000000000..6984f5cb3c --- /dev/null +++ b/codex-rs/execpolicy/src/program.rs @@ -0,0 +1,247 @@ +use serde::Serialize; +use std::collections::HashMap; +use std::collections::HashSet; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_resolver::resolve_observed_args_with_patterns; +use crate::arg_resolver::PositionalArg; +use crate::error::Error; +use crate::error::Result; +use crate::opt::Opt; +use crate::opt::OptMeta; +use crate::valid_exec::MatchedFlag; +use crate::valid_exec::MatchedOpt; +use crate::valid_exec::ValidExec; +use crate::ArgType; +use crate::ExecCall; + +#[derive(Debug)] +pub struct ProgramSpec { + pub program: String, + pub system_path: Vec, + pub option_bundling: bool, + pub combined_format: bool, + pub allowed_options: HashMap, + pub arg_patterns: Vec, + forbidden: Option, + required_options: HashSet, + should_match: Vec>, + should_not_match: Vec>, +} + +impl ProgramSpec { + pub fn new( + program: String, + system_path: Vec, + option_bundling: bool, + combined_format: bool, + allowed_options: HashMap, + arg_patterns: Vec, + forbidden: Option, + should_match: Vec>, + should_not_match: Vec>, + ) -> Self { + let required_options = allowed_options + .iter() + .filter_map(|(name, opt)| { + if opt.required { + Some(name.clone()) + } else { + None + } + }) + .collect(); + Self { + program, + system_path, + option_bundling, + combined_format, + allowed_options, + arg_patterns, + forbidden, + required_options, + should_match, + should_not_match, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub enum MatchedExec { + Match { exec: ValidExec }, + Forbidden { cause: Forbidden, reason: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub enum Forbidden { + Program { + program: String, + exec_call: ExecCall, + }, + Arg { + arg: String, + exec_call: ExecCall, + }, + Exec { + exec: ValidExec, + }, +} + +impl ProgramSpec { + // TODO(mbolin): The idea is that there should be a set of rules defined for + // a program and the args should be checked against the rules to determine + // if the program should be allowed to run. + pub fn check(&self, exec_call: &ExecCall) -> Result { + let mut expecting_option_value: Option<(String, ArgType)> = None; + let mut args = Vec::::new(); + let mut matched_flags = Vec::::new(); + let mut matched_opts = Vec::::new(); + + for (index, arg) in exec_call.args.iter().enumerate() { + if let Some(expected) = expecting_option_value { + // If we are expecting an option value, then the next argument + // should be the value for the option. + // This had better not be another option! + let (name, arg_type) = expected; + if arg.starts_with("-") { + return Err(Error::OptionFollowedByOptionInsteadOfValue { + program: self.program.clone(), + option: name, + value: arg.clone(), + }); + } + + matched_opts.push(MatchedOpt::new(&name, arg, arg_type)?); + expecting_option_value = None; + } else if arg == "--" { + return Err(Error::DoubleDashNotSupportedYet { + program: self.program.clone(), + }); + } else if arg.starts_with("-") { + match self.allowed_options.get(arg) { + Some(opt) => { + match &opt.meta { + OptMeta::Flag => { + matched_flags.push(MatchedFlag { name: arg.clone() }); + // A flag does not expect an argument: continue. + continue; + } + OptMeta::Value(arg_type) => { + expecting_option_value = Some((arg.clone(), arg_type.clone())); + continue; + } + } + } + None => { + // It could be an --option=value style flag... + } + } + + return Err(Error::UnknownOption { + program: self.program.clone(), + option: arg.clone(), + }); + } else { + args.push(PositionalArg { + index, + value: arg.clone(), + }); + } + } + + if let Some(expected) = expecting_option_value { + let (name, _arg_type) = expected; + return Err(Error::OptionMissingValue { + program: self.program.clone(), + option: name, + }); + } + + let matched_args = + resolve_observed_args_with_patterns(&self.program, args, &self.arg_patterns)?; + + // Verify all required options are present. + let matched_opt_names: HashSet = matched_opts + .iter() + .map(|opt| opt.name().to_string()) + .collect(); + if !matched_opt_names.is_superset(&self.required_options) { + let mut options = self + .required_options + .difference(&matched_opt_names) + .map(|s| s.to_string()) + .collect::>(); + options.sort(); + return Err(Error::MissingRequiredOptions { + program: self.program.clone(), + options, + }); + } + + let exec = ValidExec { + program: self.program.clone(), + flags: matched_flags, + opts: matched_opts, + args: matched_args, + system_path: self.system_path.clone(), + }; + match &self.forbidden { + Some(reason) => Ok(MatchedExec::Forbidden { + cause: Forbidden::Exec { exec }, + reason: reason.clone(), + }), + None => Ok(MatchedExec::Match { exec }), + } + } + + pub fn verify_should_match_list(&self) -> Vec { + let mut violations = Vec::new(); + for good in &self.should_match { + let exec_call = ExecCall { + program: self.program.clone(), + args: good.clone(), + }; + match self.check(&exec_call) { + Ok(_) => {} + Err(error) => { + violations.push(PositiveExampleFailedCheck { + program: self.program.clone(), + args: good.clone(), + error, + }); + } + } + } + violations + } + + pub fn verify_should_not_match_list(&self) -> Vec { + let mut violations = Vec::new(); + for bad in &self.should_not_match { + let exec_call = ExecCall { + program: self.program.clone(), + args: bad.clone(), + }; + if self.check(&exec_call).is_ok() { + violations.push(NegativeExamplePassedCheck { + program: self.program.clone(), + args: bad.clone(), + }); + } + } + violations + } +} + +#[derive(Debug, Eq, PartialEq)] +pub struct PositiveExampleFailedCheck { + pub program: String, + pub args: Vec, + pub error: Error, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct NegativeExamplePassedCheck { + pub program: String, + pub args: Vec, +} diff --git a/codex-rs/execpolicy/src/sed_command.rs b/codex-rs/execpolicy/src/sed_command.rs new file mode 100644 index 0000000000..64494ddf00 --- /dev/null +++ b/codex-rs/execpolicy/src/sed_command.rs @@ -0,0 +1,17 @@ +use crate::error::Error; +use crate::error::Result; + +pub fn parse_sed_command(sed_command: &str) -> Result<()> { + // For now, we parse only commands like `122,202p`. + if let Some(stripped) = sed_command.strip_suffix("p") { + if let Some((first, rest)) = stripped.split_once(",") { + if first.parse::().is_ok() && rest.parse::().is_ok() { + return Ok(()); + } + } + } + + Err(Error::SedCommandNotProvablySafe { + command: sed_command.to_string(), + }) +} diff --git a/codex-rs/execpolicy/src/valid_exec.rs b/codex-rs/execpolicy/src/valid_exec.rs new file mode 100644 index 0000000000..0cc3b239ca --- /dev/null +++ b/codex-rs/execpolicy/src/valid_exec.rs @@ -0,0 +1,95 @@ +use crate::arg_type::ArgType; +use crate::error::Result; +use serde::Serialize; + +/// exec() invocation that has been accepted by a `Policy`. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct ValidExec { + pub program: String, + pub flags: Vec, + pub opts: Vec, + pub args: Vec, + + /// If non-empty, a prioritized list of paths to try instead of `program`. + /// For example, `/bin/ls` is harder to compromise than whatever `ls` + /// happens to be in the user's `$PATH`, so `/bin/ls` would be included for + /// `ls`. The caller is free to disregard this list and use `program`. + pub system_path: Vec, +} + +impl ValidExec { + pub fn new(program: &str, args: Vec, system_path: &[&str]) -> Self { + Self { + program: program.to_string(), + flags: vec![], + opts: vec![], + args, + system_path: system_path.iter().map(|&s| s.to_string()).collect(), + } + } + + /// Whether a possible side effect of running this command includes writing + /// a file. + pub fn might_write_files(&self) -> bool { + self.opts.iter().any(|opt| opt.r#type.might_write_file()) + || self.args.iter().any(|opt| opt.r#type.might_write_file()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedArg { + pub index: usize, + pub r#type: ArgType, + pub value: String, +} + +impl MatchedArg { + pub fn new(index: usize, r#type: ArgType, value: &str) -> Result { + r#type.validate(value)?; + Ok(Self { + index, + r#type, + value: value.to_string(), + }) + } +} + +/// A match for an option declared with opt() in a .policy file. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedOpt { + /// Name of the option that was matched. + pub name: String, + /// Value supplied for the option. + pub value: String, + /// Type of the value supplied for the option. + pub r#type: ArgType, +} + +impl MatchedOpt { + pub fn new(name: &str, value: &str, r#type: ArgType) -> Result { + r#type.validate(value)?; + Ok(Self { + name: name.to_string(), + value: value.to_string(), + r#type, + }) + } + + pub fn name(&self) -> &str { + &self.name + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedFlag { + /// Name of the flag that was matched. + pub name: String, +} + +impl MatchedFlag { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + } + } +} diff --git a/codex-rs/execpolicy/tests/bad.rs b/codex-rs/execpolicy/tests/bad.rs new file mode 100644 index 0000000000..91f8b52ba4 --- /dev/null +++ b/codex-rs/execpolicy/tests/bad.rs @@ -0,0 +1,9 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::NegativeExamplePassedCheck; + +#[test] +fn verify_everything_in_bad_list_is_rejected() { + let policy = get_default_policy().expect("failed to load default policy"); + let violations = policy.check_each_bad_list_individually(); + assert_eq!(Vec::::new(), violations); +} diff --git a/codex-rs/execpolicy/tests/cp.rs b/codex-rs/execpolicy/tests/cp.rs new file mode 100644 index 0000000000..8981ac7a34 --- /dev/null +++ b/codex-rs/execpolicy/tests/cp.rs @@ -0,0 +1,85 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgMatcher; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_cp_no_args() { + let policy = setup(); + let cp = ExecCall::new("cp", &[]); + assert_eq!( + Err(Error::NotEnoughArgs { + program: "cp".to_string(), + args: vec![], + arg_patterns: vec![ArgMatcher::ReadableFiles, ArgMatcher::WriteableFile] + }), + policy.check(&cp) + ) +} + +#[test] +fn test_cp_one_arg() { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo/bar"]); + + assert_eq!( + Err(Error::VarargMatcherDidNotMatchAnything { + program: "cp".to_string(), + matcher: ArgMatcher::ReadableFiles, + }), + policy.check(&cp) + ); +} + +#[test] +fn test_cp_one_file() -> Result<()> { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo/bar", "../baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "cp", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo/bar")?, + MatchedArg::new(1, ArgType::WriteableFile, "../baz")?, + ], + &["/bin/cp", "/usr/bin/cp"] + ) + }), + policy.check(&cp) + ); + Ok(()) +} + +#[test] +fn test_cp_multiple_files() -> Result<()> { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "cp", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo")?, + MatchedArg::new(1, ArgType::ReadableFile, "bar")?, + MatchedArg::new(2, ArgType::WriteableFile, "baz")?, + ], + &["/bin/cp", "/usr/bin/cp"] + ) + }), + policy.check(&cp) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/good.rs b/codex-rs/execpolicy/tests/good.rs new file mode 100644 index 0000000000..18a002850c --- /dev/null +++ b/codex-rs/execpolicy/tests/good.rs @@ -0,0 +1,9 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::PositiveExampleFailedCheck; + +#[test] +fn verify_everything_in_good_list_is_allowed() { + let policy = get_default_policy().expect("failed to load default policy"); + let violations = policy.check_each_good_list_individually(); + assert_eq!(Vec::::new(), violations); +} diff --git a/codex-rs/execpolicy/tests/head.rs b/codex-rs/execpolicy/tests/head.rs new file mode 100644 index 0000000000..196de081f6 --- /dev/null +++ b/codex-rs/execpolicy/tests/head.rs @@ -0,0 +1,132 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgMatcher; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedOpt; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +extern crate codex_execpolicy; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_head_no_args() { + let policy = setup(); + let head = ExecCall::new("head", &[]); + // It is actually valid to call `head` without arguments: it will read from + // stdin instead of from a file. Though recall that a command rejected by + // the policy is not "unsafe:" it just means that this library cannot + // *guarantee* that the command is safe. + // + // If we start verifying individual components of a shell command, such as: + // `find . -name | head -n 10`, then it might be important to allow the + // no-arg case. + assert_eq!( + Err(Error::VarargMatcherDidNotMatchAnything { + program: "head".to_string(), + matcher: ArgMatcher::ReadableFiles, + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_one_file_no_flags() -> Result<()> { + let policy = setup(); + let head = ExecCall::new("head", &["src/extension.ts"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "head", + vec![MatchedArg::new( + 0, + ArgType::ReadableFile, + "src/extension.ts" + )?], + &["/bin/head", "/usr/bin/head"] + ) + }), + policy.check(&head) + ); + Ok(()) +} + +#[test] +fn test_head_one_flag_one_file() -> Result<()> { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "100", "src/extension.ts"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "head".to_string(), + flags: vec![], + opts: vec![MatchedOpt::new("-n", "100", ArgType::PositiveInteger).unwrap()], + args: vec![MatchedArg::new( + 2, + ArgType::ReadableFile, + "src/extension.ts" + )?], + system_path: vec!["/bin/head".to_string(), "/usr/bin/head".to_string()], + } + }), + policy.check(&head) + ); + Ok(()) +} + +#[test] +fn test_head_invalid_n_as_0() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "0", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "0".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_nonint_float() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "1.5", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "1.5".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_float() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "1.0", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "1.0".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_negative_int() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "-1", "src/extension.ts"]); + assert_eq!( + Err(Error::OptionFollowedByOptionInsteadOfValue { + program: "head".to_string(), + option: "-n".to_string(), + value: "-1".to_string(), + }), + policy.check(&head) + ) +} diff --git a/codex-rs/execpolicy/tests/literal.rs b/codex-rs/execpolicy/tests/literal.rs new file mode 100644 index 0000000000..d849371e3b --- /dev/null +++ b/codex-rs/execpolicy/tests/literal.rs @@ -0,0 +1,50 @@ +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::PolicyParser; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +extern crate codex_execpolicy; + +#[test] +fn test_invalid_subcommand() -> Result<()> { + let unparsed_policy = r#" +define_program( + program="fake_executable", + args=["subcommand", "sub-subcommand"], +) +"#; + let parser = PolicyParser::new("test_invalid_subcommand", unparsed_policy); + let policy = parser.parse().expect("failed to parse policy"); + let valid_call = ExecCall::new("fake_executable", &["subcommand", "sub-subcommand"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "fake_executable", + vec![ + MatchedArg::new(0, ArgType::Literal("subcommand".to_string()), "subcommand")?, + MatchedArg::new( + 1, + ArgType::Literal("sub-subcommand".to_string()), + "sub-subcommand" + )?, + ], + &[] + ) + }), + policy.check(&valid_call) + ); + + let invalid_call = ExecCall::new("fake_executable", &["subcommand", "not-a-real-subcommand"]); + assert_eq!( + Err(Error::LiteralValueDidNotMatch { + expected: "sub-subcommand".to_string(), + actual: "not-a-real-subcommand".to_string() + }), + policy.check(&invalid_call) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/ls.rs b/codex-rs/execpolicy/tests/ls.rs new file mode 100644 index 0000000000..f7e78f22f3 --- /dev/null +++ b/codex-rs/execpolicy/tests/ls.rs @@ -0,0 +1,166 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_ls_no_args() { + let policy = setup(); + let ls = ExecCall::new("ls", &[]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new("ls", vec![], &["/bin/ls", "/usr/bin/ls"]) + }), + policy.check(&ls) + ); +} + +#[test] +fn test_ls_dash_a_dash_l() { + let policy = setup(); + let args = &["-a", "-l"]; + let ls_a_l = ExecCall::new("ls", args); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-a"), MatchedFlag::new("-l")], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_a_l) + ); +} + +#[test] +fn test_ls_dash_z() { + let policy = setup(); + + // -z is currently an invalid option for ls, but it has so many options, + // perhaps it will get added at some point... + let ls_z = ExecCall::new("ls", &["-z"]); + assert_eq!( + Err(Error::UnknownOption { + program: "ls".into(), + option: "-z".into() + }), + policy.check(&ls_z) + ); +} + +#[test] +fn test_ls_dash_al() { + let policy = setup(); + + // This currently fails, but it should pass once option_bundling=True is implemented. + let ls_al = ExecCall::new("ls", &["-al"]); + assert_eq!( + Err(Error::UnknownOption { + program: "ls".into(), + option: "-al".into() + }), + policy.check(&ls_al) + ); +} + +#[test] +fn test_ls_one_file_arg() -> Result<()> { + let policy = setup(); + + let ls_one_file_arg = ExecCall::new("ls", &["foo"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "ls", + vec![MatchedArg::new(0, ArgType::ReadableFile, "foo")?], + &["/bin/ls", "/usr/bin/ls"] + ) + }), + policy.check(&ls_one_file_arg) + ); + Ok(()) +} + +#[test] +fn test_ls_multiple_file_args() -> Result<()> { + let policy = setup(); + + let ls_multiple_file_args = ExecCall::new("ls", &["foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "ls", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo")?, + MatchedArg::new(1, ArgType::ReadableFile, "bar")?, + MatchedArg::new(2, ArgType::ReadableFile, "baz")?, + ], + &["/bin/ls", "/usr/bin/ls"] + ) + }), + policy.check(&ls_multiple_file_args) + ); + Ok(()) +} + +#[test] +fn test_ls_multiple_flags_and_file_args() -> Result<()> { + let policy = setup(); + + let ls_multiple_flags_and_file_args = ExecCall::new("ls", &["-l", "-a", "foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-l"), MatchedFlag::new("-a")], + args: vec![ + MatchedArg::new(2, ArgType::ReadableFile, "foo")?, + MatchedArg::new(3, ArgType::ReadableFile, "bar")?, + MatchedArg::new(4, ArgType::ReadableFile, "baz")?, + ], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_multiple_flags_and_file_args) + ); + Ok(()) +} + +#[test] +fn test_flags_after_file_args() -> Result<()> { + let policy = setup(); + + // TODO(mbolin): While this is "safe" in that it will not do anything bad + // to the user's machine, it will fail because apparently `ls` does not + // allow flags after file arguments (as some commands do). We should + // extend define_program() to make this part of the configuration so that + // this command is disallowed. + let ls_flags_after_file_args = ExecCall::new("ls", &["foo", "-l"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-l")], + args: vec![MatchedArg::new(0, ArgType::ReadableFile, "foo")?], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_flags_after_file_args) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/parse_sed_command.rs b/codex-rs/execpolicy/tests/parse_sed_command.rs new file mode 100644 index 0000000000..6d03b626ef --- /dev/null +++ b/codex-rs/execpolicy/tests/parse_sed_command.rs @@ -0,0 +1,23 @@ +use codex_execpolicy::parse_sed_command; +use codex_execpolicy::Error; + +#[test] +fn parses_simple_print_command() { + assert_eq!(parse_sed_command("122,202p"), Ok(())); +} + +#[test] +fn rejects_malformed_print_command() { + assert_eq!( + parse_sed_command("122,202"), + Err(Error::SedCommandNotProvablySafe { + command: "122,202".to_string(), + }) + ); + assert_eq!( + parse_sed_command("122202"), + Err(Error::SedCommandNotProvablySafe { + command: "122202".to_string(), + }) + ); +} diff --git a/codex-rs/execpolicy/tests/pwd.rs b/codex-rs/execpolicy/tests/pwd.rs new file mode 100644 index 0000000000..4e29e4cbc1 --- /dev/null +++ b/codex-rs/execpolicy/tests/pwd.rs @@ -0,0 +1,85 @@ +extern crate codex_execpolicy; + +use std::vec; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::Policy; +use codex_execpolicy::PositionalArg; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_pwd_no_args() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &[]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_capital_l() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["-L"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + flags: vec![MatchedFlag::new("-L")], + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_capital_p() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["-P"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + flags: vec![MatchedFlag::new("-P")], + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_extra_args() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["foo", "bar"]); + assert_eq!( + Err(Error::UnexpectedArguments { + program: "pwd".to_string(), + args: vec![ + PositionalArg { + index: 0, + value: "foo".to_string() + }, + PositionalArg { + index: 1, + value: "bar".to_string() + }, + ], + }), + policy.check(&pwd) + ); +} diff --git a/codex-rs/execpolicy/tests/sed.rs b/codex-rs/execpolicy/tests/sed.rs new file mode 100644 index 0000000000..cc26bf1eb4 --- /dev/null +++ b/codex-rs/execpolicy/tests/sed.rs @@ -0,0 +1,83 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::MatchedOpt; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_sed_print_specific_lines() -> Result<()> { + let policy = setup(); + let sed = ExecCall::new("sed", &["-n", "122,202p", "hello.txt"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "sed".to_string(), + flags: vec![MatchedFlag::new("-n")], + args: vec![ + MatchedArg::new(1, ArgType::SedCommand, "122,202p")?, + MatchedArg::new(2, ArgType::ReadableFile, "hello.txt")?, + ], + system_path: vec!["/usr/bin/sed".to_string()], + ..Default::default() + } + }), + policy.check(&sed) + ); + Ok(()) +} + +#[test] +fn test_sed_print_specific_lines_with_e_flag() -> Result<()> { + let policy = setup(); + let sed = ExecCall::new("sed", &["-n", "-e", "122,202p", "hello.txt"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "sed".to_string(), + flags: vec![MatchedFlag::new("-n")], + opts: vec![MatchedOpt::new("-e", "122,202p", ArgType::SedCommand).unwrap()], + args: vec![MatchedArg::new(3, ArgType::ReadableFile, "hello.txt")?], + system_path: vec!["/usr/bin/sed".to_string()], + } + }), + policy.check(&sed) + ); + Ok(()) +} + +#[test] +fn test_sed_reject_dangerous_command() { + let policy = setup(); + let sed = ExecCall::new("sed", &["-e", "s/y/echo hi/e", "hello.txt"]); + assert_eq!( + Err(Error::SedCommandNotProvablySafe { + command: "s/y/echo hi/e".to_string(), + }), + policy.check(&sed) + ); +} + +#[test] +fn test_sed_verify_e_or_pattern_is_required() { + let policy = setup(); + let sed = ExecCall::new("sed", &["122,202p"]); + assert_eq!( + Err(Error::MissingRequiredOptions { + program: "sed".to_string(), + options: vec!["-e".to_string()], + }), + policy.check(&sed) + ); +} From 687dc8b68b88509eb1171cbde95be58e0894e65b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 15:03:56 -0700 Subject: [PATCH 56/84] feat: introduce codex_execpolicy crate for defining "safe" commands --- codex-rs/Cargo.lock | 1063 ++++++++++++++++- codex-rs/Cargo.toml | 1 + codex-rs/execpolicy/Cargo.toml | 28 + codex-rs/execpolicy/README.md | 180 +++ codex-rs/execpolicy/build.rs | 3 + codex-rs/execpolicy/src/arg_matcher.rs | 118 ++ codex-rs/execpolicy/src/arg_resolver.rs | 194 +++ codex-rs/execpolicy/src/arg_type.rs | 87 ++ codex-rs/execpolicy/src/default.policy | 202 ++++ codex-rs/execpolicy/src/error.rs | 96 ++ codex-rs/execpolicy/src/exec_call.rs | 28 + codex-rs/execpolicy/src/execv_checker.rs | 263 ++++ codex-rs/execpolicy/src/lib.rs | 45 + codex-rs/execpolicy/src/main.rs | 166 +++ codex-rs/execpolicy/src/opt.rs | 77 ++ codex-rs/execpolicy/src/policy.rs | 103 ++ codex-rs/execpolicy/src/policy_parser.rs | 222 ++++ codex-rs/execpolicy/src/program.rs | 247 ++++ codex-rs/execpolicy/src/sed_command.rs | 17 + codex-rs/execpolicy/src/valid_exec.rs | 95 ++ codex-rs/execpolicy/tests/bad.rs | 9 + codex-rs/execpolicy/tests/cp.rs | 85 ++ codex-rs/execpolicy/tests/good.rs | 9 + codex-rs/execpolicy/tests/head.rs | 132 ++ codex-rs/execpolicy/tests/literal.rs | 50 + codex-rs/execpolicy/tests/ls.rs | 166 +++ .../execpolicy/tests/parse_sed_command.rs | 23 + codex-rs/execpolicy/tests/pwd.rs | 85 ++ codex-rs/execpolicy/tests/sed.rs | 83 ++ 29 files changed, 3830 insertions(+), 47 deletions(-) create mode 100644 codex-rs/execpolicy/Cargo.toml create mode 100644 codex-rs/execpolicy/README.md create mode 100644 codex-rs/execpolicy/build.rs create mode 100644 codex-rs/execpolicy/src/arg_matcher.rs create mode 100644 codex-rs/execpolicy/src/arg_resolver.rs create mode 100644 codex-rs/execpolicy/src/arg_type.rs create mode 100644 codex-rs/execpolicy/src/default.policy create mode 100644 codex-rs/execpolicy/src/error.rs create mode 100644 codex-rs/execpolicy/src/exec_call.rs create mode 100644 codex-rs/execpolicy/src/execv_checker.rs create mode 100644 codex-rs/execpolicy/src/lib.rs create mode 100644 codex-rs/execpolicy/src/main.rs create mode 100644 codex-rs/execpolicy/src/opt.rs create mode 100644 codex-rs/execpolicy/src/policy.rs create mode 100644 codex-rs/execpolicy/src/policy_parser.rs create mode 100644 codex-rs/execpolicy/src/program.rs create mode 100644 codex-rs/execpolicy/src/sed_command.rs create mode 100644 codex-rs/execpolicy/src/valid_exec.rs create mode 100644 codex-rs/execpolicy/tests/bad.rs create mode 100644 codex-rs/execpolicy/tests/cp.rs create mode 100644 codex-rs/execpolicy/tests/good.rs create mode 100644 codex-rs/execpolicy/tests/head.rs create mode 100644 codex-rs/execpolicy/tests/literal.rs create mode 100644 codex-rs/execpolicy/tests/ls.rs create mode 100644 codex-rs/execpolicy/tests/parse_sed_command.rs create mode 100644 codex-rs/execpolicy/tests/pwd.rs create mode 100644 codex-rs/execpolicy/tests/sed.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f9f5860861..1f91c0072b 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "addr2line" version = "0.21.0" @@ -17,6 +27,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy 0.7.35", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -26,6 +48,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocative" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fac2ce611db8b8cee9b2aa886ca03c924e9da5e5295d0dbd0526e5d0b0710f7" +dependencies = [ + "allocative_derive", + "bumpalo", + "ctor", + "hashbrown 0.14.5", + "num-bigint", +] + +[[package]] +name = "allocative_derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -47,6 +93,15 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "ansi-to-tui" version = "7.0.0" @@ -128,6 +183,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -174,7 +238,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -222,6 +286,33 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.9.0" @@ -262,6 +353,12 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.10.1" @@ -298,6 +395,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.40" @@ -308,6 +411,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -331,7 +435,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", "terminal_size", ] @@ -344,7 +448,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -353,6 +457,21 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "clipboard-win" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmp_any" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9b18233253483ce2f65329a24072ec414db782531bdbb7d0bbc4bd2ce6b7e21" + [[package]] name = "codex-ansi-escape" version = "0.1.0" @@ -445,6 +564,26 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "codex-execpolicy" +version = "0.1.0" +dependencies = [ + "allocative", + "anyhow", + "clap", + "derive_more", + "env_logger", + "log", + "multimap", + "path-absolutize", + "regex", + "serde", + "serde_json", + "serde_with", + "starlark", + "tempfile", +] + [[package]] name = "codex-interactive" version = "0.1.0" @@ -551,6 +690,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -588,7 +736,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.9.0", "crossterm_winapi", "mio", "parking_lot", @@ -607,6 +755,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "darling" version = "0.20.11" @@ -627,8 +791,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.100", ] [[package]] @@ -639,7 +803,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -660,6 +824,17 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "debugserver-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf6834a70ed14e8e4e41882df27190bea150f1f6ecf461f1033f8739cd8af4a" +dependencies = [ + "schemafy", + "serde", + "serde_json", +] + [[package]] name = "deranged" version = "0.4.0" @@ -667,6 +842,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn 2.0.100", + "unicode-xid", ] [[package]] @@ -701,6 +910,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.5.0" @@ -713,6 +932,27 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users 0.4.6", + "winapi", +] + +[[package]] +name = "display_container" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a110a75c96bedec8e65823dea00a1d710288b7a369d95fd8a0f5127639466fa" +dependencies = [ + "either", + "indenter", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -721,7 +961,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -730,12 +970,41 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dupe" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed2bc011db9c93fbc2b6cdb341a53737a55bafb46dbb74cf6764fc33a2fbf9c" +dependencies = [ + "dupe_derive", +] + +[[package]] +name = "dupe_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e195b4945e88836d826124af44fdcb262ec01ef94d44f14f4fb5103f19892a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "ena" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +dependencies = [ + "log", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -745,6 +1014,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enumflags2" version = "0.7.11" @@ -762,7 +1037,7 @@ checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -771,12 +1046,44 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbfd0e7fc632dec5e6c9396a27bc9f9975b4e039720e1fd3e34021d3ce28c415" +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + [[package]] name = "errno" version = "0.3.11" @@ -787,6 +1094,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "error-code" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" + [[package]] name = "event-listener" version = "5.4.0" @@ -846,6 +1159,23 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix 1.0.5", + "windows-sys 0.59.0", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.10.0" @@ -956,7 +1286,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -989,6 +1319,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1041,13 +1380,29 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1071,6 +1426,27 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "1.3.1" @@ -1330,7 +1706,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1366,6 +1742,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.9.0" @@ -1373,7 +1760,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", + "serde", ] [[package]] @@ -1392,7 +1780,16 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn", + "syn 2.0.100", +] + +[[package]] +name = "inventory" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab08d7cd2c5897f2c949e5383ea7c7db03fb19130ffcfbf7eda795137ae3cb83" +dependencies = [ + "rustversion", ] [[package]] @@ -1401,12 +1798,32 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1422,6 +1839,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a064218214dc6a10fbae5ec5fa888d80c45d611aba169222fc272072bf7aef6" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1432,6 +1873,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lalrpop" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b" +dependencies = [ + "ascii-canvas", + "bit-set", + "diff", + "ena", + "is-terminal", + "itertools 0.10.5", + "lalrpop-util", + "petgraph", + "regex", + "regex-syntax 0.6.29", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", +] + +[[package]] +name = "lalrpop-util" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed" +dependencies = [ + "regex", +] + [[package]] name = "landlock" version = "0.4.1" @@ -1461,7 +1933,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags", + "bitflags 2.9.0", "libc", ] @@ -1499,15 +1971,57 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "logos" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8b031682c67a8e3d5446840f9573eb7fe26efe7ec8d195c9ac4c0647c502f1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 1.0.109", +] + [[package]] name = "lru" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown", + "hashbrown 0.15.2", ] +[[package]] +name = "lsp-types" +version = "0.94.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66bfd44a06ae10647fe3f8214762e9369fd4248df1350924b4ef9e770a85ea1" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.1.0" @@ -1523,6 +2037,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1566,6 +2089,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +dependencies = [ + "serde", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -1583,6 +2115,33 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1620,12 +2179,31 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1641,7 +2219,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -1666,7 +2244,7 @@ version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ - "bitflags", + "bitflags 2.9.0", "cfg-if", "foreign-types", "libc", @@ -1683,7 +2261,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -1784,12 +2362,49 @@ dependencies = [ "nom_locate", ] +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.9.0", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1808,6 +2423,21 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1820,9 +2450,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.24", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "predicates" version = "3.1.3" @@ -1897,6 +2533,16 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.9.1" @@ -1932,13 +2578,13 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags", + "bitflags 2.9.0", "cassowary", "compact_str", "crossterm", "indoc", "instability", - "itertools", + "itertools 0.13.0", "lru", "paste", "strum", @@ -1959,7 +2605,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -1973,6 +2619,17 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.0" @@ -1984,6 +2641,26 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "regex" version = "1.11.1" @@ -2112,7 +2789,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2125,7 +2802,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ - "bitflags", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.9.4", @@ -2177,6 +2854,28 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "rustyline" +version = "14.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "clipboard-win", + "fd-lock", + "home", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "unicode-segmentation", + "unicode-width 0.1.14", + "utf8parse", + "windows-sys 0.52.0", +] + [[package]] name = "ryu" version = "1.0.20" @@ -2192,6 +2891,48 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemafy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aea5ba40287dae331f2c48b64dbc8138541f5e97ee8793caa7948c1f31d86d5" +dependencies = [ + "Inflector", + "schemafy_core", + "schemafy_lib", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "syn 1.0.109", +] + +[[package]] +name = "schemafy_core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41781ae092f4fd52c9287efb74456aea0d3b90032d2ecad272bd14dbbcb0511b" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "schemafy_lib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e953db32579999ca98c451d80801b6f6a7ecba6127196c5387ec0774c528befa" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "schemafy_core", + "serde", + "serde_derive", + "serde_json", + "syn 1.0.109", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2213,7 +2954,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.9.0", "core-foundation", "core-foundation-sys", "libc", @@ -2247,7 +2988,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2256,13 +2997,24 @@ version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ - "indexmap", + "indexmap 2.9.0", "itoa", "memchr", "ryu", "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -2284,6 +3036,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.9.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2341,6 +3123,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -2372,6 +3160,96 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "starlark" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f53849859f05d9db705b221bd92eede93877fd426c1b4a3c3061403a5912a8f" +dependencies = [ + "allocative", + "anyhow", + "bumpalo", + "cmp_any", + "debugserver-types", + "derivative", + "derive_more", + "display_container", + "dupe", + "either", + "erased-serde", + "hashbrown 0.14.5", + "inventory", + "itertools 0.13.0", + "maplit", + "memoffset", + "num-bigint", + "num-traits", + "once_cell", + "paste", + "ref-cast", + "regex", + "rustyline", + "serde", + "serde_json", + "starlark_derive", + "starlark_map", + "starlark_syntax", + "static_assertions", + "strsim 0.10.0", + "textwrap", + "thiserror 1.0.69", +] + +[[package]] +name = "starlark_derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe58bc6c8b7980a1fe4c9f8f48200c3212db42ebfe21ae6a0336385ab53f082a" +dependencies = [ + "dupe", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "starlark_map" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92659970f120df0cc1c0bb220b33587b7a9a90e80d4eecc5c5af5debb950173d" +dependencies = [ + "allocative", + "dupe", + "equivalent", + "fxhash", + "hashbrown 0.14.5", + "serde", +] + +[[package]] +name = "starlark_syntax" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe53b3690d776aafd7cb6b9fed62d94f83280e3b87d88e3719cc0024638461b3" +dependencies = [ + "allocative", + "annotate-snippets", + "anyhow", + "derivative", + "derive_more", + "dupe", + "lalrpop", + "lalrpop-util", + "logos", + "lsp-types", + "memchr", + "num-bigint", + "num-traits", + "once_cell", + "starlark_map", + "thiserror 1.0.69", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2384,6 +3262,24 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2409,7 +3305,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.100", ] [[package]] @@ -2418,6 +3314,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.100" @@ -2446,7 +3353,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2455,7 +3362,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags", + "bitflags 2.9.0", "core-foundation", "system-configuration-sys", ] @@ -2483,6 +3390,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "terminal_size" version = "0.4.2" @@ -2499,6 +3417,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2525,7 +3452,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2536,7 +3463,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2580,6 +3507,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2615,7 +3551,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2678,7 +3614,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", @@ -2744,7 +3680,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -2877,7 +3813,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ - "itertools", + "itertools 0.13.0", "unicode-segmentation", "unicode-width 0.1.14", ] @@ -2894,6 +3830,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -2909,6 +3851,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2941,6 +3884,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -3002,7 +3951,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -3037,7 +3986,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3117,7 +4066,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3128,7 +4077,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3360,7 +4309,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags", + "bitflags 2.9.0", ] [[package]] @@ -3401,17 +4350,37 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive 0.7.35", +] + [[package]] name = "zerocopy" version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.8.24", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -3422,7 +4391,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] [[package]] @@ -3442,7 +4411,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", "synstructure", ] @@ -3471,5 +4440,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.100", ] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index f3f66eb2d7..69c4e8a8a0 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -6,6 +6,7 @@ members = [ "cli", "core", "exec", + "execpolicy", "interactive", "repl", "tui", diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml new file mode 100644 index 0000000000..6d8fd5ac05 --- /dev/null +++ b/codex-rs/execpolicy/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "codex-execpolicy" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "codex-execpolicy" +path = "src/main.rs" + +[lib] +name = "codex_execpolicy" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +starlark = "0.13.0" +allocative = "0.3.3" +clap = { version = "4", features = ["derive"] } +derive_more = { version = "1", features = ["display"] } +env_logger = "0.11.5" +log = "0.4" +multimap = "0.10.0" +path-absolutize = "3.1.1" +regex = "1.11.1" +serde = { version = "1.0.194", features = ["derive"] } +serde_json = "1.0.110" +serde_with = { version = "3", features = ["macros"] } +tempfile = "3.13.0" diff --git a/codex-rs/execpolicy/README.md b/codex-rs/execpolicy/README.md new file mode 100644 index 0000000000..ca95829440 --- /dev/null +++ b/codex-rs/execpolicy/README.md @@ -0,0 +1,180 @@ +# codex_execpolicy + +The goal of this library is to classify a proposed [`execv(3)`](https://linux.die.net/man/3/execv) command into one of the following states: + +- `safe` The command is safe to run (\*). +- `match` The command matched a rule in the policy, but the caller should decide whether it is safe to run based on the files it will write. +- `forbidden` The command is not allowed to be run. +- `unverified` The safety cannot be determined: make the user decide. + +(\*) Whether an `execv(3)` call should be considered "safe" often requires additional context beyond the arguments to `execv()` itself. For example, if you trust an autonomous software agent to write files in your source tree, then deciding whether `/bin/cp foo bar` is "safe" depends on `getcwd(3)` for the calling process as well as the `realpath` of `foo` and `bar` when resolved against `getcwd()`. +To that end, rather than returning a boolean, the validator returns a structured result that the client is expected to use to determine the "safety" of the proposed `execv()` call. + +For example, to check the command `ls -l foo`, the checker would be invoked as follows: + +```shell +cargo run -- check ls -l foo | jq +``` + +It will exit with `0` and print the following to stdout: + +```json +{ + "result": "safe", + "match": { + "program": "ls", + "flags": [ + { + "name": "-l" + } + ], + "opts": [], + "args": [ + { + "index": 1, + "type": "ReadableFile", + "value": "foo" + } + ], + "system_path": ["/bin/ls", "/usr/bin/ls"] + } +} +``` + +Of note: + +- `foo` is tagged as a `ReadableFile`, so the caller should resolve `foo` relative to `getcwd()` and `realpath` it (as it may be a symlink) to determine whether `foo` is safe to read. +- While the specified executable is `ls`, `"system_path"` offers `/bin/ls` and `/usr/bin/ls` as viable alternatives to avoid using whatever `ls` happens to appear first on the user's `$PATH`. If either exists on the host, it is recommended to use it as the first argument to `execv(3)` instead of `ls`. + +Further, "safety" in this system is not a guarantee that the command will execute successfully. As an example, `cat /Users/mbolin/code/codex/README.md` may be considered "safe" if the system has decided the agent is allowed to read anything under `/Users/mbolin/code/codex`, but it will fail at runtime if `README.md` does not exist. (Though this is "safe" in that the agent did not read any files that it was not authorized to read.) + +## Policy + +Currently, the default policy is defined in [`default.policy`](./src/default.policy) within the crate. + +The system uses [Starlark](https://bazel.build/rules/language) as the file format because, unlike something like JSON or YAML, it supports "macros" without compromising on safety or reproducibility. (Under the hood, we use [`starlark-rust`](https://github.com/facebook/starlark-rust) as the specific Starlark implementation.) + +This policy contains "rules" such as: + +```python +define_program( + program="cp", + options=[ + flag("-r"), + flag("-R"), + flag("--recursive"), + ], + args=[ARG_RFILES, ARG_WFILE], + system_path=["/bin/cp", "/usr/bin/cp"], + should_match=[ + ["foo", "bar"], + ], + should_not_match=[ + ["foo"], + ], +) +``` + +This rule means that: + +- `cp` can be used with any of the following flags (where "flag" means "an option that does not take an argument"): `-r`, `-R`, `--recursive`. +- The initial `ARG_RFILES` passed to `args` means that it expects one or more arguments that correspond to "readable files" +- The final `ARG_WFILE` passed to `args` means that it expects exactly one argument that corresponds to a "writeable file." +- As a means of a lightweight way of including a unit test alongside the definition, the `should_match` list is a list of examples of `execv(3)` args that should match the rule and `should_not_match` is a list of examples that should not match. These examples are verified when the `.policy` file is loaded. + +Note that the language of the `.policy` file is still evolving, as we have to continue to expand it so it is sufficiently expressive to accept all commands we want to consider "safe" without allowing unsafe commands to pass through. + +The integrity of `default.policy` is verified [via unit tests](./tests). + +Further, the CLI supports a `--policy` option to specify a custom `.policy` file for ad-hoc testing. + +## Output Type: `match` + +Going back to the `cp` example, because the rule matches an `ARG_WFILE`, it will return `match` instead of `safe`: + +```shell +cargo run -- check cp src1 src2 dest | jq +``` + +If the caller wants to consider allowing this command, it should parse the JSON to pick out the `WriteableFile` arguments and decide whether they are safe to write: + +```json +{ + "result": "match", + "match": { + "program": "cp", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": "ReadableFile", + "value": "src1" + }, + { + "index": 1, + "type": "ReadableFile", + "value": "src2" + }, + { + "index": 2, + "type": "WriteableFile", + "value": "dest" + } + ], + "system_path": ["/bin/cp", "/usr/bin/cp"] + } +} +``` + +Note the exit code is still `0` for a `match` unless the `--require-safe` flag is specified, in which case the exit code is `12`. + +## Output Type: `forbidden` + +It is also possible to define a rule that, if it matches a command, should flag it as _forbidden_. For example, we do not want agents to be able to run `applied deploy` _ever_, so we define the following rule: + +```python +define_program( + program="applied", + args=["deploy"], + forbidden="Infrastructure Risk: command contains 'applied deploy'", + should_match=[ + ["deploy"], + ], + should_not_match=[ + ["lint"], + ], +) +``` + +Note that for a rule to be forbidden, the `forbidden` keyword arg must be specified as the reason the command is forbidden. This will be included in the output: + +```shell +cargo run -- check applied deploy | jq +``` + +```json +{ + "result": "forbidden", + "reason": "Infrastructure Risk: command contains 'applied deploy'", + "cause": { + "Exec": { + "exec": { + "program": "applied", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": { + "Literal": "deploy" + }, + "value": "deploy" + } + ], + "system_path": [] + } + } + } +} +``` diff --git a/codex-rs/execpolicy/build.rs b/codex-rs/execpolicy/build.rs new file mode 100644 index 0000000000..eda4846853 --- /dev/null +++ b/codex-rs/execpolicy/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=src/default.policy"); +} diff --git a/codex-rs/execpolicy/src/arg_matcher.rs b/codex-rs/execpolicy/src/arg_matcher.rs new file mode 100644 index 0000000000..12d91b4465 --- /dev/null +++ b/codex-rs/execpolicy/src/arg_matcher.rs @@ -0,0 +1,118 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::arg_type::ArgType; +use crate::starlark::values::ValueLike; +use allocative::Allocative; +use derive_more::derive::Display; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::string::StarlarkStr; +use starlark::values::AllocValue; +use starlark::values::Heap; +use starlark::values::NoSerialize; +use starlark::values::StarlarkValue; +use starlark::values::UnpackValue; +use starlark::values::Value; + +/// Patterns that lists of arguments should be compared against. +#[derive(Clone, Debug, Display, Eq, PartialEq, NoSerialize, ProvidesStaticType, Allocative)] +#[display("{}", self)] +pub enum ArgMatcher { + /// Literal string value. + Literal(String), + + /// We cannot say what type of value this should match, but it is *not* a file path. + OpaqueNonFile, + + /// Required readable file. + ReadableFile, + + /// Required writeable file. + WriteableFile, + + /// Non-empty list of readable files. + ReadableFiles, + + /// Non-empty list of readable files, or empty list, implying readable cwd. + ReadableFilesOrCwd, + + /// Positive integer, like one that is required for `head -n`. + PositiveInteger, + + /// Bespoke matcher for safe sed commands. + SedCommand, + + /// Matches an arbitrary number of arguments without attributing any + /// particular meaning to them. Caller is responsible for interpreting them. + UnverifiedVarargs, +} + +impl ArgMatcher { + pub fn cardinality(&self) -> ArgMatcherCardinality { + match self { + ArgMatcher::Literal(_) + | ArgMatcher::OpaqueNonFile + | ArgMatcher::ReadableFile + | ArgMatcher::WriteableFile + | ArgMatcher::PositiveInteger + | ArgMatcher::SedCommand => ArgMatcherCardinality::One, + ArgMatcher::ReadableFiles => ArgMatcherCardinality::AtLeastOne, + ArgMatcher::ReadableFilesOrCwd | ArgMatcher::UnverifiedVarargs => { + ArgMatcherCardinality::ZeroOrMore + } + } + } + + pub fn arg_type(&self) -> ArgType { + match self { + ArgMatcher::Literal(value) => ArgType::Literal(value.clone()), + ArgMatcher::OpaqueNonFile => ArgType::OpaqueNonFile, + ArgMatcher::ReadableFile => ArgType::ReadableFile, + ArgMatcher::WriteableFile => ArgType::WriteableFile, + ArgMatcher::ReadableFiles => ArgType::ReadableFile, + ArgMatcher::ReadableFilesOrCwd => ArgType::ReadableFile, + ArgMatcher::PositiveInteger => ArgType::PositiveInteger, + ArgMatcher::SedCommand => ArgType::SedCommand, + ArgMatcher::UnverifiedVarargs => ArgType::Unknown, + } + } +} + +pub enum ArgMatcherCardinality { + One, + AtLeastOne, + ZeroOrMore, +} + +impl ArgMatcherCardinality { + pub fn is_exact(&self) -> Option { + match self { + ArgMatcherCardinality::One => Some(1), + ArgMatcherCardinality::AtLeastOne => None, + ArgMatcherCardinality::ZeroOrMore => None, + } + } +} + +impl<'v> AllocValue<'v> for ArgMatcher { + fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + heap.alloc_simple(self) + } +} + +#[starlark_value(type = "ArgMatcher")] +impl<'v> StarlarkValue<'v> for ArgMatcher { + type Canonical = ArgMatcher; +} + +impl<'v> UnpackValue<'v> for ArgMatcher { + type Error = starlark::Error; + + fn unpack_value_impl(value: Value<'v>) -> starlark::Result> { + if let Some(str) = value.downcast_ref::() { + Ok(Some(ArgMatcher::Literal(str.as_str().to_string()))) + } else { + Ok(value.downcast_ref::().cloned()) + } + } +} diff --git a/codex-rs/execpolicy/src/arg_resolver.rs b/codex-rs/execpolicy/src/arg_resolver.rs new file mode 100644 index 0000000000..d1138a8ffc --- /dev/null +++ b/codex-rs/execpolicy/src/arg_resolver.rs @@ -0,0 +1,194 @@ +use serde::Serialize; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_matcher::ArgMatcherCardinality; +use crate::error::Error; +use crate::error::Result; +use crate::valid_exec::MatchedArg; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct PositionalArg { + pub index: usize, + pub value: String, +} + +pub fn resolve_observed_args_with_patterns( + program: &str, + args: Vec, + arg_patterns: &Vec, +) -> Result> { + // Naive matching implementation. Among `arg_patterns`, there is allowed to + // be at most one vararg pattern. Assuming `arg_patterns` is non-empty, we + // end up with either: + // + // - all `arg_patterns` in `prefix_patterns` + // - `arg_patterns` split across `prefix_patterns` (which could be empty), + // one `vararg_pattern`, and `suffix_patterns` (which could also empty). + // + // From there, we start by matching everything in `prefix_patterns`. + // Then we calculate how many positional args should be matched by + // `suffix_patterns` and use that to determine how many args are left to + // be matched by `vararg_pattern` (which could be zero). + // + // After assocating positional args with `vararg_pattern`, we match the + // `suffix_patterns` with the remaining args. + let ParitionedArgs { + num_prefix_args, + num_suffix_args, + prefix_patterns, + suffix_patterns, + vararg_pattern, + } = partition_args(program, arg_patterns)?; + + let mut matched_args = Vec::::new(); + + let prefix = get_range_checked(&args, 0..num_prefix_args)?; + let mut prefix_arg_index = 0; + for pattern in prefix_patterns { + let n = pattern.cardinality().is_exact().unwrap(); + for positional_arg in &prefix[prefix_arg_index..prefix_arg_index + n] { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + prefix_arg_index += n; + } + + if num_suffix_args > args.len() { + return Err(Error::NotEnoughArgs { + program: program.to_string(), + args, + arg_patterns: arg_patterns.clone(), + }); + } + + let initial_suffix_args_index = args.len() - num_suffix_args; + if prefix_arg_index > initial_suffix_args_index { + return Err(Error::PrefixOverlapsSuffix {}); + } + + if let Some(pattern) = vararg_pattern { + let vararg = get_range_checked(&args, prefix_arg_index..initial_suffix_args_index)?; + match pattern.cardinality() { + ArgMatcherCardinality::One => { + return Err(Error::InternalInvariantViolation { + message: "vararg pattern should not have cardinality of one".to_string(), + }); + } + ArgMatcherCardinality::AtLeastOne => { + if vararg.is_empty() { + return Err(Error::VarargMatcherDidNotMatchAnything { + program: program.to_string(), + matcher: pattern, + }); + } else { + for positional_arg in vararg { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + } + } + ArgMatcherCardinality::ZeroOrMore => { + for positional_arg in vararg { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + } + } + } + + let suffix = get_range_checked(&args, initial_suffix_args_index..args.len())?; + let mut suffix_arg_index = 0; + for pattern in suffix_patterns { + let n = pattern.cardinality().is_exact().unwrap(); + for positional_arg in &suffix[suffix_arg_index..suffix_arg_index + n] { + let matched_arg = MatchedArg::new( + positional_arg.index, + pattern.arg_type(), + &positional_arg.value.clone(), + )?; + matched_args.push(matched_arg); + } + suffix_arg_index += n; + } + + if matched_args.len() < args.len() { + let extra_args = get_range_checked(&args, matched_args.len()..args.len())?; + Err(Error::UnexpectedArguments { + program: program.to_string(), + args: extra_args.to_vec(), + }) + } else { + Ok(matched_args) + } +} + +#[derive(Default)] +struct ParitionedArgs { + num_prefix_args: usize, + num_suffix_args: usize, + prefix_patterns: Vec, + suffix_patterns: Vec, + vararg_pattern: Option, +} + +fn partition_args(program: &str, arg_patterns: &Vec) -> Result { + let mut in_prefix = true; + let mut partitioned_args = ParitionedArgs::default(); + + for pattern in arg_patterns { + match pattern.cardinality().is_exact() { + Some(n) => { + if in_prefix { + partitioned_args.prefix_patterns.push(pattern.clone()); + partitioned_args.num_prefix_args += n; + } else { + partitioned_args.suffix_patterns.push(pattern.clone()); + partitioned_args.num_suffix_args += n; + } + } + None => match partitioned_args.vararg_pattern { + None => { + partitioned_args.vararg_pattern = Some(pattern.clone()); + in_prefix = false; + } + Some(existing_pattern) => { + return Err(Error::MultipleVarargPatterns { + program: program.to_string(), + first: existing_pattern, + second: pattern.clone(), + }); + } + }, + } + } + + Ok(partitioned_args) +} + +fn get_range_checked(vec: &[T], range: std::ops::Range) -> Result<&[T]> { + if range.start > range.end { + Err(Error::RangeStartExceedsEnd { + start: range.start, + end: range.end, + }) + } else if range.end > vec.len() { + Err(Error::RangeEndOutOfBounds { + end: range.end, + len: vec.len(), + }) + } else { + Ok(&vec[range]) + } +} diff --git a/codex-rs/execpolicy/src/arg_type.rs b/codex-rs/execpolicy/src/arg_type.rs new file mode 100644 index 0000000000..11be0277ec --- /dev/null +++ b/codex-rs/execpolicy/src/arg_type.rs @@ -0,0 +1,87 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::error::Error; +use crate::error::Result; +use crate::sed_command::parse_sed_command; +use allocative::Allocative; +use derive_more::derive::Display; +use serde::Serialize; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::StarlarkValue; + +#[derive(Debug, Clone, Display, Eq, PartialEq, ProvidesStaticType, Allocative, Serialize)] +#[display("{}", self)] +pub enum ArgType { + Literal(String), + /// We cannot say what this argument represents, but it is *not* a file path. + OpaqueNonFile, + /// A file (or directory) that can be expected to be read as part of this command. + ReadableFile, + /// A file (or directory) that can be expected to be written as part of this command. + WriteableFile, + /// Positive integer, like one that is required for `head -n`. + PositiveInteger, + /// Bespoke arg type for a safe sed command. + SedCommand, + /// Type is unknown: it may or may not be a file. + Unknown, +} + +impl ArgType { + pub fn validate(&self, value: &str) -> Result<()> { + match self { + ArgType::Literal(literal_value) => { + if value != *literal_value { + Err(Error::LiteralValueDidNotMatch { + expected: literal_value.clone(), + actual: value.to_string(), + }) + } else { + Ok(()) + } + } + ArgType::ReadableFile => { + if value.is_empty() { + Err(Error::EmptyFileName {}) + } else { + Ok(()) + } + } + ArgType::WriteableFile => { + if value.is_empty() { + Err(Error::EmptyFileName {}) + } else { + Ok(()) + } + } + ArgType::OpaqueNonFile | ArgType::Unknown => Ok(()), + ArgType::PositiveInteger => match value.parse::() { + Ok(0) => Err(Error::InvalidPositiveInteger { + value: value.to_string(), + }), + Ok(_) => Ok(()), + Err(_) => Err(Error::InvalidPositiveInteger { + value: value.to_string(), + }), + }, + ArgType::SedCommand => parse_sed_command(value), + } + } + + pub fn might_write_file(&self) -> bool { + match self { + ArgType::WriteableFile | ArgType::Unknown => true, + ArgType::Literal(_) + | ArgType::OpaqueNonFile + | ArgType::PositiveInteger + | ArgType::ReadableFile + | ArgType::SedCommand => false, + } + } +} + +#[starlark_value(type = "ArgType")] +impl<'v> StarlarkValue<'v> for ArgType { + type Canonical = ArgType; +} diff --git a/codex-rs/execpolicy/src/default.policy b/codex-rs/execpolicy/src/default.policy new file mode 100644 index 0000000000..bd27a0bb30 --- /dev/null +++ b/codex-rs/execpolicy/src/default.policy @@ -0,0 +1,202 @@ +""" +define_program() supports the following arguments: +- program: the name of the program +- system_path: list of absolute paths on the system where program can likely be found +- option_bundling (PLANNED): whether to allow bundling of options (e.g. `-al` for `-a -l`) +- combine_format (PLANNED): whether to allow `--option=value` (as opposed to `--option value`) +- options: the command-line flags/options: use flag() and opt() to define these +- args: the rules for what arguments are allowed that are not "options" +- should_match: list of command-line invocations that should be matched by the rule +- should_not_match: list of command-line invocations that should not be matched by the rule +""" + +define_program( + program="ls", + system_path=["/bin/ls", "/usr/bin/ls"], + options=[ + flag("-1"), + flag("-a"), + flag("-l"), + ], + args=[ARG_RFILES_OR_CWD], +) + +define_program( + program="cat", + options=[ + flag("-b"), + flag("-n"), + flag("-t"), + ], + system_path=["/bin/cat", "/usr/bin/cat"], + args=[ARG_RFILES], + should_match=[ + ["file.txt"], + ["-n", "file.txt"], + ["-b", "file.txt"], + ], + should_not_match=[ + # While cat without args is valid, it will read from stdin, which + # does not seem appropriate for our current use case. + [], + # Let's not auto-approve advisory locking. + ["-l", "file.txt"], + ] +) + +define_program( + program="cp", + options=[ + flag("-r"), + flag("-R"), + flag("--recursive"), + ], + args=[ARG_RFILES, ARG_WFILE], + system_path=["/bin/cp", "/usr/bin/cp"], + should_match=[ + ["foo", "bar"], + ], + should_not_match=[ + ["foo"], + ], +) + +define_program( + program="head", + system_path=["/bin/head", "/usr/bin/head"], + options=[ + opt("-c", ARG_POS_INT), + opt("-n", ARG_POS_INT), + ], + args=[ARG_RFILES], +) + +printenv_system_path = ["/usr/bin/printenv"] + +# Print all environment variables. +define_program( + program="printenv", + args=[], + system_path=printenv_system_path, + # This variant of `printenv` only allows zero args. + should_match=[[]], + should_not_match=[["PATH"]], +) + +# Print a specific environment variable. +define_program( + program="printenv", + args=[ARG_OPAQUE_VALUE], + system_path=printenv_system_path, + # This variant of `printenv` only allows exactly one arg. + should_match=[["PATH"]], + should_not_match=[[], ["PATH", "HOME"]], +) + +# Note that `pwd` is generally implemented as a shell built-in. It does not +# accept any arguments. +define_program( + program="pwd", + options=[ + flag("-L"), + flag("-P"), + ], + args=[], +) + +define_program( + program="rg", + options=[ + opt("-A", ARG_POS_INT), + opt("-B", ARG_POS_INT), + opt("-C", ARG_POS_INT), + opt("-d", ARG_POS_INT), + opt("--max-depth", ARG_POS_INT), + opt("-g", ARG_OPAQUE_VALUE), + opt("--glob", ARG_OPAQUE_VALUE), + opt("-m", ARG_POS_INT), + opt("--max-count", ARG_POS_INT), + + flag("-n"), + flag("-i"), + flag("-l"), + flag("--files"), + flag("--files-with-matches"), + flag("--files-without-match"), + ], + args=[ARG_OPAQUE_VALUE, ARG_RFILES_OR_CWD], + should_match=[ + ["-n", "init"], + ["-n", "init", "."], + ["-i", "-n", "init", "src"], + ["--files", "--max-depth", "2", "."], + ], + should_not_match=[ + ["-m", "-n", "init"], + ["--glob", "src"], + ], + # TODO(mbolin): Perhaps we need a way to indicate that we expect `rg` to be + # bundled with the host environment and we should be using that verison. + system_path=[], +) + +# Unfortunately, `sed` is difficult to secure because GNU sed supports an `e` +# flag where `s/pattern/replacement/e` would run `replacement` as a shell +# command every time `pattern` is matched. For example, try the following on +# Ubuntu (which uses GNU sed, unlike macOS): +# +# ```shell +# $ yes | head -n 4 > /tmp/yes.txt +# $ sed 's/y/echo hi/e' /tmp/yes.txt +# hi +# hi +# hi +# hi +# ``` +# +# As you can see, `echo hi` got executed four times. In order to support some +# basic sed functionality, we implement a bespoke `ARG_SED_COMMAND` that matches +# only "known safe" sed commands. +common_sed_flags = [ + # We deliberately do not support -i or -f. + flag("-n"), + flag("-u"), +] +sed_system_path = ["/usr/bin/sed"] + +# When -e is not specified, the first argument must be a valid sed command. +define_program( + program="sed", + options=common_sed_flags, + args=[ARG_SED_COMMAND, ARG_RFILES], + system_path=sed_system_path, +) + +# When -e is required, all arguments are assumed to be readable files. +define_program( + program="sed", + options=common_sed_flags + [ + opt("-e", ARG_SED_COMMAND, required=True), + ], + args=[ARG_RFILES], + system_path=sed_system_path, +) + +define_program( + program="which", + options=[ + flag("-a"), + flag("-s"), + ], + # Surprisingly, `which` takes more than one argument. + args=[ARG_RFILES], + should_match=[ + ["python3"], + ["-a", "python3"], + ["-a", "python3", "cargo"], + ], + should_not_match=[ + [], + ], + system_path=["/bin/which", "/usr/bin/which"], +) diff --git a/codex-rs/execpolicy/src/error.rs b/codex-rs/execpolicy/src/error.rs new file mode 100644 index 0000000000..ff781f43a5 --- /dev/null +++ b/codex-rs/execpolicy/src/error.rs @@ -0,0 +1,96 @@ +use std::path::PathBuf; + +use serde::Serialize; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_resolver::PositionalArg; +use serde_with::serde_as; +use serde_with::DisplayFromStr; + +pub type Result = std::result::Result; + +#[serde_as] +#[derive(Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "type")] +pub enum Error { + NoSpecForProgram { + program: String, + }, + OptionMissingValue { + program: String, + option: String, + }, + OptionFollowedByOptionInsteadOfValue { + program: String, + option: String, + value: String, + }, + UnknownOption { + program: String, + option: String, + }, + UnexpectedArguments { + program: String, + args: Vec, + }, + DoubleDashNotSupportedYet { + program: String, + }, + MultipleVarargPatterns { + program: String, + first: ArgMatcher, + second: ArgMatcher, + }, + RangeStartExceedsEnd { + start: usize, + end: usize, + }, + RangeEndOutOfBounds { + end: usize, + len: usize, + }, + PrefixOverlapsSuffix {}, + NotEnoughArgs { + program: String, + args: Vec, + arg_patterns: Vec, + }, + InternalInvariantViolation { + message: String, + }, + VarargMatcherDidNotMatchAnything { + program: String, + matcher: ArgMatcher, + }, + EmptyFileName {}, + LiteralValueDidNotMatch { + expected: String, + actual: String, + }, + InvalidPositiveInteger { + value: String, + }, + MissingRequiredOptions { + program: String, + options: Vec, + }, + SedCommandNotProvablySafe { + command: String, + }, + ReadablePathNotInReadableFolders { + file: PathBuf, + folders: Vec, + }, + WriteablePathNotInWriteableFolders { + file: PathBuf, + folders: Vec, + }, + CannotCheckRelativePath { + file: PathBuf, + }, + CannotCanonicalizePath { + file: String, + #[serde_as(as = "DisplayFromStr")] + error: std::io::ErrorKind, + }, +} diff --git a/codex-rs/execpolicy/src/exec_call.rs b/codex-rs/execpolicy/src/exec_call.rs new file mode 100644 index 0000000000..e9753eccf3 --- /dev/null +++ b/codex-rs/execpolicy/src/exec_call.rs @@ -0,0 +1,28 @@ +use std::fmt::Display; + +use serde::Serialize; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ExecCall { + pub program: String, + pub args: Vec, +} + +impl ExecCall { + pub fn new(program: &str, args: &[&str]) -> Self { + Self { + program: program.to_string(), + args: args.iter().map(|&s| s.into()).collect(), + } + } +} + +impl Display for ExecCall { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.program)?; + for arg in &self.args { + write!(f, " {}", arg)?; + } + Ok(()) + } +} diff --git a/codex-rs/execpolicy/src/execv_checker.rs b/codex-rs/execpolicy/src/execv_checker.rs new file mode 100644 index 0000000000..787fbce122 --- /dev/null +++ b/codex-rs/execpolicy/src/execv_checker.rs @@ -0,0 +1,263 @@ +use std::ffi::OsString; +use std::path::Path; +use std::path::PathBuf; + +use crate::ArgType; +use crate::Error::CannotCanonicalizePath; +use crate::Error::CannotCheckRelativePath; +use crate::Error::ReadablePathNotInReadableFolders; +use crate::Error::WriteablePathNotInWriteableFolders; +use crate::ExecCall; +use crate::MatchedExec; +use crate::Policy; +use crate::Result; +use crate::ValidExec; +use path_absolutize::*; +use std::os::unix::fs::PermissionsExt; + +macro_rules! check_file_in_folders { + ($file:expr, $folders:expr, $error:ident) => { + if !$folders.iter().any(|folder| $file.starts_with(folder)) { + return Err($error { + file: $file.clone(), + folders: $folders.to_vec(), + }); + } + }; +} + +pub struct ExecvChecker { + execv_policy: Policy, +} + +impl ExecvChecker { + pub fn new(execv_policy: Policy) -> Self { + Self { execv_policy } + } + + pub fn r#match(&self, exec_call: &ExecCall) -> Result { + self.execv_policy.check(exec_call) + } + + /// The caller is responsible for ensuring readable_folders and + /// writeable_folders are in canonical form. + pub fn check( + &self, + valid_exec: ValidExec, + cwd: &Option, + readable_folders: &[PathBuf], + writeable_folders: &[PathBuf], + ) -> Result { + for (arg_type, value) in valid_exec + .args + .into_iter() + .map(|arg| (arg.r#type, arg.value)) + .chain( + valid_exec + .opts + .into_iter() + .map(|opt| (opt.r#type, opt.value)), + ) + { + match arg_type { + ArgType::ReadableFile => { + let readable_file = ensure_absolute_path(&value, cwd)?; + check_file_in_folders!( + readable_file, + readable_folders, + ReadablePathNotInReadableFolders + ); + } + ArgType::WriteableFile => { + let writeable_file = ensure_absolute_path(&value, cwd)?; + check_file_in_folders!( + writeable_file, + writeable_folders, + WriteablePathNotInWriteableFolders + ); + } + ArgType::OpaqueNonFile + | ArgType::Unknown + | ArgType::PositiveInteger + | ArgType::SedCommand + | ArgType::Literal(_) => { + continue; + } + } + } + + let mut program = valid_exec.program.to_string(); + for system_path in valid_exec.system_path { + if is_executable_file(&system_path) { + program = system_path.to_string(); + break; + } + } + + Ok(program) + } +} + +fn ensure_absolute_path(path: &str, cwd: &Option) -> Result { + let file = PathBuf::from(path); + let result = if file.is_relative() { + match cwd { + Some(cwd) => file.absolutize_from(cwd), + None => return Err(CannotCheckRelativePath { file }), + } + } else { + file.absolutize() + }; + result + .map(|path| path.into_owned()) + .map_err(|error| CannotCanonicalizePath { + file: path.to_string(), + error: error.kind(), + }) +} + +fn is_executable_file(path: &str) -> bool { + let file_path = Path::new(path); + + if let Ok(metadata) = std::fs::metadata(file_path) { + let permissions = metadata.permissions(); + // Check if the file is executable (by checking the executable bit for the owner) + return metadata.is_file() && (permissions.mode() & 0o111 != 0); + } + + false +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + use crate::MatchedArg; + use crate::PolicyParser; + + fn setup(fake_cp: &Path) -> ExecvChecker { + let source = format!( + r#" +define_program( +program="cp", +args=[ARG_RFILE, ARG_WFILE], +system_path=[{fake_cp:?}] +) +"# + ); + let parser = PolicyParser::new("#test", &source); + let policy = parser.parse().unwrap(); + ExecvChecker::new(policy) + } + + #[test] + fn test_check_valid_input_files() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + + // Create an executable file that can be used with the system_path arg. + let fake_cp = temp_dir.path().join("cp"); + let fake_cp_file = std::fs::File::create(&fake_cp).unwrap(); + let mut permissions = fake_cp_file.metadata().unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&fake_cp, permissions).unwrap(); + + // Create root_path and reference to files under the root. + let root_path = temp_dir.path().to_path_buf(); + let source_path = root_path.join("source"); + let dest_path = root_path.join("dest"); + + let cp = fake_cp.to_str().unwrap().to_string(); + let root = root_path.to_str().unwrap().to_string(); + let source = source_path.to_str().unwrap().to_string(); + let dest = dest_path.to_str().unwrap().to_string(); + + let cwd = Some(root_path.clone().into()); + + let checker = setup(&fake_cp); + let exec_call = ExecCall { + program: "cp".into(), + args: vec![source.clone(), dest.clone()], + }; + let valid_exec = match checker.r#match(&exec_call)? { + MatchedExec::Match { exec } => exec, + unexpected => panic!("Expected a safe exec but got {unexpected:?}"), + }; + + // No readable or writeable folders specified. + assert_eq!( + checker.check(valid_exec.clone(), &cwd, &[], &[]), + Err(ReadablePathNotInReadableFolders { + file: source_path.clone(), + folders: vec![] + }), + ); + + // Only readable folders specified. + assert_eq!( + checker.check(valid_exec.clone(), &cwd, &[root_path.clone()], &[]), + Err(WriteablePathNotInWriteableFolders { + file: dest_path.clone(), + folders: vec![] + }), + ); + + // Both readable and writeable folders specified. + assert_eq!( + checker.check( + valid_exec.clone(), + &cwd, + &[root_path.clone()], + &[root_path.clone()] + ), + Ok(cp.clone()), + ); + + // Args are the readable and writeable folders, not files within the + // folders. + let exec_call_folders_as_args = ExecCall { + program: "cp".into(), + args: vec![root.clone(), root.clone()], + }; + let valid_exec_call_folders_as_args = match checker.r#match(&exec_call_folders_as_args)? { + MatchedExec::Match { exec } => exec, + _ => panic!("Expected a safe exec"), + }; + assert_eq!( + checker.check( + valid_exec_call_folders_as_args, + &cwd, + &[root_path.clone()], + &[root_path.clone()] + ), + Ok(cp.clone()), + ); + + // Specify a parent of a readable folder as input. + let exec_with_parent_of_readable_folder = ValidExec { + program: "cp".into(), + args: vec![ + MatchedArg::new( + 0, + ArgType::ReadableFile, + root_path.parent().unwrap().to_str().unwrap(), + )?, + MatchedArg::new(1, ArgType::WriteableFile, &dest)?, + ], + ..Default::default() + }; + assert_eq!( + checker.check( + exec_with_parent_of_readable_folder, + &cwd, + &[root_path.clone()], + &[dest_path.clone()] + ), + Err(ReadablePathNotInReadableFolders { + file: root_path.parent().unwrap().to_path_buf(), + folders: vec![root_path.clone()] + }), + ); + Ok(()) + } +} diff --git a/codex-rs/execpolicy/src/lib.rs b/codex-rs/execpolicy/src/lib.rs new file mode 100644 index 0000000000..6f12225981 --- /dev/null +++ b/codex-rs/execpolicy/src/lib.rs @@ -0,0 +1,45 @@ +#![allow(clippy::type_complexity)] +#![allow(clippy::too_many_arguments)] +#[macro_use] +extern crate starlark; + +mod arg_matcher; +mod arg_resolver; +mod arg_type; +mod error; +mod exec_call; +mod execv_checker; +mod opt; +mod policy; +mod policy_parser; +mod program; +mod sed_command; +mod valid_exec; + +pub use arg_matcher::ArgMatcher; +pub use arg_resolver::PositionalArg; +pub use arg_type::ArgType; +pub use error::Error; +pub use error::Result; +pub use exec_call::ExecCall; +pub use execv_checker::ExecvChecker; +pub use opt::Opt; +pub use policy::Policy; +pub use policy_parser::PolicyParser; +pub use program::Forbidden; +pub use program::MatchedExec; +pub use program::NegativeExamplePassedCheck; +pub use program::PositiveExampleFailedCheck; +pub use program::ProgramSpec; +pub use sed_command::parse_sed_command; +pub use valid_exec::MatchedArg; +pub use valid_exec::MatchedFlag; +pub use valid_exec::MatchedOpt; +pub use valid_exec::ValidExec; + +const DEFAULT_POLICY: &str = include_str!("default.policy"); + +pub fn get_default_policy() -> starlark::Result { + let parser = PolicyParser::new("#default", DEFAULT_POLICY); + parser.parse() +} diff --git a/codex-rs/execpolicy/src/main.rs b/codex-rs/execpolicy/src/main.rs new file mode 100644 index 0000000000..d8cb034d2a --- /dev/null +++ b/codex-rs/execpolicy/src/main.rs @@ -0,0 +1,166 @@ +use anyhow::Result; +use clap::Parser; +use clap::Subcommand; +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::Policy; +use codex_execpolicy::PolicyParser; +use codex_execpolicy::ValidExec; +use serde::de; +use serde::Deserialize; +use serde::Serialize; +use std::path::PathBuf; +use std::str::FromStr; + +const MATCHED_BUT_WRITES_FILES_EXIT_CODE: i32 = 12; +const MIGHT_BE_SAFE_EXIT_CODE: i32 = 13; +const FORBIDDEN_EXIT_CODE: i32 = 14; + +#[derive(Parser, Deserialize, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + /// If the command fails the policy, exit with 13, but print parseable JSON + /// to stdout. + #[clap(long)] + pub require_safe: bool, + + /// Path to the policy file. + #[clap(long, short = 'p')] + pub policy: Option, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Clone, Debug, Deserialize, Subcommand)] +pub enum Command { + /// Checks the command as if the arguments were the inputs to execv(3). + Check { + #[arg(trailing_var_arg = true)] + command: Vec, + }, + + /// Checks the command encoded as a JSON object. + #[clap(name = "check-json")] + CheckJson { + /// JSON object with "program" (str) and "args" (list[str]) fields. + #[serde(deserialize_with = "deserialize_from_json")] + exec: ExecArg, + }, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ExecArg { + pub program: String, + + #[serde(default)] + pub args: Vec, +} + +fn main() -> Result<()> { + env_logger::init(); + + let args = Args::parse(); + let policy = match args.policy { + Some(policy) => { + let policy_source = policy.to_string_lossy().to_string(); + let unparsed_policy = std::fs::read_to_string(policy)?; + let parser = PolicyParser::new(&policy_source, &unparsed_policy); + parser.parse() + } + None => get_default_policy(), + }; + let policy = policy.map_err(|err| err.into_anyhow())?; + + let exec = match args.command { + Command::Check { command } => match command.split_first() { + Some((first, rest)) => ExecArg { + program: first.to_string(), + args: rest.iter().map(|s| s.to_string()).collect(), + }, + None => { + eprintln!("no command provided"); + std::process::exit(1); + } + }, + Command::CheckJson { exec } => exec, + }; + + let (output, exit_code) = check_command(&policy, exec, args.require_safe); + let json = serde_json::to_string(&output)?; + println!("{}", json); + std::process::exit(exit_code); +} + +fn check_command( + policy: &Policy, + ExecArg { program, args }: ExecArg, + check: bool, +) -> (Output, i32) { + let exec_call = ExecCall { program, args }; + match policy.check(&exec_call) { + Ok(MatchedExec::Match { exec }) => { + if exec.might_write_files() { + let exit_code = if check { + MATCHED_BUT_WRITES_FILES_EXIT_CODE + } else { + 0 + }; + (Output::Match { r#match: exec }, exit_code) + } else { + (Output::Safe { r#match: exec }, 0) + } + } + Ok(MatchedExec::Forbidden { reason, cause }) => { + let exit_code = if check { FORBIDDEN_EXIT_CODE } else { 0 }; + (Output::Forbidden { reason, cause }, exit_code) + } + Err(err) => { + let exit_code = if check { MIGHT_BE_SAFE_EXIT_CODE } else { 0 }; + (Output::Unverified { error: err }, exit_code) + } + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "result")] +pub enum Output { + /// The command is verified as safe. + #[serde(rename = "safe")] + Safe { r#match: ValidExec }, + + /// The command has matched a rule in the policy, but the caller should + /// decide whether it is "safe" given the files it wants to write. + #[serde(rename = "match")] + Match { r#match: ValidExec }, + + /// The user is forbidden from running the command. + #[serde(rename = "forbidden")] + Forbidden { + reason: String, + cause: codex_execpolicy::Forbidden, + }, + + /// The safety of the command could not be verified. + #[serde(rename = "unverified")] + Unverified { error: codex_execpolicy::Error }, +} + +fn deserialize_from_json<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let decoded = serde_json::from_str(&s) + .map_err(|e| serde::de::Error::custom(format!("JSON parse error: {e}")))?; + Ok(decoded) +} + +impl FromStr for ExecArg { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + serde_json::from_str(s).map_err(|e| e.into()) + } +} diff --git a/codex-rs/execpolicy/src/opt.rs b/codex-rs/execpolicy/src/opt.rs new file mode 100644 index 0000000000..4a58037462 --- /dev/null +++ b/codex-rs/execpolicy/src/opt.rs @@ -0,0 +1,77 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::starlark::values::ValueLike; +use crate::ArgType; +use allocative::Allocative; +use derive_more::derive::Display; +use starlark::any::ProvidesStaticType; +use starlark::values::starlark_value; +use starlark::values::AllocValue; +use starlark::values::Heap; +use starlark::values::NoSerialize; +use starlark::values::StarlarkValue; +use starlark::values::UnpackValue; +use starlark::values::Value; + +/// Command line option that takes a value. +#[derive(Clone, Debug, Display, PartialEq, Eq, ProvidesStaticType, NoSerialize, Allocative)] +#[display("opt({})", opt)] +pub struct Opt { + /// The option as typed on the command line, e.g., `-h` or `--help`. If + /// it can be used in the `--name=value` format, then this should be + /// `--name` (though this is subject to change). + pub opt: String, + pub meta: OptMeta, + pub required: bool, +} + +/// When defining an Opt, use as specific an OptMeta as possible. +#[derive(Clone, Debug, Display, PartialEq, Eq, ProvidesStaticType, NoSerialize, Allocative)] +#[display("{}", self)] +pub enum OptMeta { + /// Option does not take a value. + Flag, + + /// Option takes a single value matching the specified type. + Value(ArgType), +} + +impl Opt { + pub fn new(opt: String, meta: OptMeta, required: bool) -> Self { + Self { + opt, + meta, + required, + } + } + + pub fn name(&self) -> &str { + &self.opt + } +} + +#[starlark_value(type = "Opt")] +impl<'v> StarlarkValue<'v> for Opt { + type Canonical = Opt; +} + +impl<'v> UnpackValue<'v> for Opt { + type Error = starlark::Error; + + fn unpack_value_impl(value: Value<'v>) -> starlark::Result> { + // TODO(mbolin): It fels like this should be doable without cloning? + // Cannot simply consume the value? + Ok(value.downcast_ref::().cloned()) + } +} + +impl<'v> AllocValue<'v> for Opt { + fn alloc_value(self, heap: &'v Heap) -> Value<'v> { + heap.alloc_simple(self) + } +} + +#[starlark_value(type = "OptMeta")] +impl<'v> StarlarkValue<'v> for OptMeta { + type Canonical = OptMeta; +} diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs new file mode 100644 index 0000000000..5ce7d7b917 --- /dev/null +++ b/codex-rs/execpolicy/src/policy.rs @@ -0,0 +1,103 @@ +use multimap::MultiMap; +use regex::Error as RegexError; +use regex::Regex; + +use crate::error::Error; +use crate::error::Result; +use crate::policy_parser::ForbiddenProgramRegex; +use crate::program::PositiveExampleFailedCheck; +use crate::ExecCall; +use crate::Forbidden; +use crate::MatchedExec; +use crate::NegativeExamplePassedCheck; +use crate::ProgramSpec; + +pub struct Policy { + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings_pattern: Option, +} + +impl Policy { + pub fn new( + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings: Vec, + ) -> std::result::Result { + let forbidden_substrings_pattern = if forbidden_substrings.is_empty() { + None + } else { + let escaped_substrings = forbidden_substrings + .iter() + .map(|s| regex::escape(s)) + .collect::>() + .join("|"); + Some(Regex::new(&format!("({escaped_substrings})"))?) + }; + Ok(Self { + programs, + forbidden_program_regexes, + forbidden_substrings_pattern, + }) + } + + pub fn check(&self, exec_call: &ExecCall) -> Result { + let ExecCall { program, args } = &exec_call; + for ForbiddenProgramRegex { regex, reason } in &self.forbidden_program_regexes { + if regex.is_match(program) { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Program { + program: program.clone(), + exec_call: exec_call.clone(), + }, + reason: reason.clone(), + }); + } + } + + for arg in args { + if let Some(regex) = &self.forbidden_substrings_pattern { + if regex.is_match(arg) { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Arg { + arg: arg.clone(), + exec_call: exec_call.clone(), + }, + reason: format!("arg `{}` contains forbidden substring", arg), + }); + } + } + } + + let mut last_err = Err(Error::NoSpecForProgram { + program: program.clone(), + }); + if let Some(spec_list) = self.programs.get_vec(program) { + for spec in spec_list { + match spec.check(exec_call) { + Ok(matched_exec) => return Ok(matched_exec), + Err(err) => { + last_err = Err(err); + } + } + } + } + last_err + } + + pub fn check_each_good_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_match_list()); + } + violations + } + + pub fn check_each_bad_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_not_match_list()); + } + violations + } +} diff --git a/codex-rs/execpolicy/src/policy_parser.rs b/codex-rs/execpolicy/src/policy_parser.rs new file mode 100644 index 0000000000..caf4efd10d --- /dev/null +++ b/codex-rs/execpolicy/src/policy_parser.rs @@ -0,0 +1,222 @@ +#![allow(clippy::needless_lifetimes)] + +use crate::arg_matcher::ArgMatcher; +use crate::opt::OptMeta; +use crate::Opt; +use crate::Policy; +use crate::ProgramSpec; +use log::info; +use multimap::MultiMap; +use regex::Regex; +use starlark::any::ProvidesStaticType; +use starlark::environment::GlobalsBuilder; +use starlark::environment::LibraryExtension; +use starlark::environment::Module; +use starlark::eval::Evaluator; +use starlark::syntax::AstModule; +use starlark::syntax::Dialect; +use starlark::values::list::UnpackList; +use starlark::values::none::NoneType; +use starlark::values::Heap; +use std::cell::RefCell; +use std::collections::HashMap; + +pub struct PolicyParser { + policy_source: String, + unparsed_policy: String, +} + +impl PolicyParser { + pub fn new(policy_source: &str, unparsed_policy: &str) -> Self { + Self { + policy_source: policy_source.to_string(), + unparsed_policy: unparsed_policy.to_string(), + } + } + + pub fn parse(&self) -> starlark::Result { + let mut dialect = Dialect::Extended.clone(); + dialect.enable_f_strings = true; + let ast = AstModule::parse(&self.policy_source, self.unparsed_policy.clone(), &dialect)?; + let globals = GlobalsBuilder::extended_by(&[LibraryExtension::Typing]) + .with(policy_builtins) + .build(); + let module = Module::new(); + + let heap = Heap::new(); + + module.set("ARG_OPAQUE_VALUE", heap.alloc(ArgMatcher::OpaqueNonFile)); + module.set("ARG_RFILE", heap.alloc(ArgMatcher::ReadableFile)); + module.set("ARG_WFILE", heap.alloc(ArgMatcher::WriteableFile)); + module.set("ARG_RFILES", heap.alloc(ArgMatcher::ReadableFiles)); + module.set( + "ARG_RFILES_OR_CWD", + heap.alloc(ArgMatcher::ReadableFilesOrCwd), + ); + module.set("ARG_POS_INT", heap.alloc(ArgMatcher::PositiveInteger)); + module.set("ARG_SED_COMMAND", heap.alloc(ArgMatcher::SedCommand)); + module.set( + "ARG_UNVERIFIED_VARARGS", + heap.alloc(ArgMatcher::UnverifiedVarargs), + ); + + let policy_builder = PolicyBuilder::new(); + { + let mut eval = Evaluator::new(&module); + eval.extra = Some(&policy_builder); + eval.eval_module(ast, &globals)?; + } + let policy = policy_builder.build(); + policy.map_err(|e| starlark::Error::new_kind(starlark::ErrorKind::Other(e.into()))) + } +} + +#[derive(Debug)] +pub struct ForbiddenProgramRegex { + pub regex: regex::Regex, + pub reason: String, +} + +#[derive(Debug, ProvidesStaticType)] +struct PolicyBuilder { + programs: RefCell>, + forbidden_program_regexes: RefCell>, + forbidden_substrings: RefCell>, +} + +impl PolicyBuilder { + fn new() -> Self { + Self { + programs: RefCell::new(MultiMap::new()), + forbidden_program_regexes: RefCell::new(Vec::new()), + forbidden_substrings: RefCell::new(Vec::new()), + } + } + + fn build(self) -> Result { + let programs = self.programs.into_inner(); + let forbidden_program_regexes = self.forbidden_program_regexes.into_inner(); + let forbidden_substrings = self.forbidden_substrings.into_inner(); + Policy::new(programs, forbidden_program_regexes, forbidden_substrings) + } + + fn add_program_spec(&self, program_spec: ProgramSpec) { + info!("adding program spec: {:?}", program_spec); + let name = program_spec.program.clone(); + let mut programs = self.programs.borrow_mut(); + programs.insert(name.clone(), program_spec); + } + + fn add_forbidden_substrings(&self, substrings: &[String]) { + let mut forbidden_substrings = self.forbidden_substrings.borrow_mut(); + forbidden_substrings.extend_from_slice(substrings); + } + + fn add_forbidden_program_regex(&self, regex: Regex, reason: String) { + let mut forbidden_program_regexes = self.forbidden_program_regexes.borrow_mut(); + forbidden_program_regexes.push(ForbiddenProgramRegex { regex, reason }); + } +} + +#[starlark_module] +fn policy_builtins(builder: &mut GlobalsBuilder) { + fn define_program<'v>( + program: String, + system_path: Option>, + option_bundling: Option, + combined_format: Option, + options: Option>, + args: Option>, + forbidden: Option, + should_match: Option>>, + should_not_match: Option>>, + eval: &mut Evaluator, + ) -> anyhow::Result { + let option_bundling = option_bundling.unwrap_or(false); + let system_path = system_path.map_or_else(Vec::new, |v| v.items.to_vec()); + let combined_format = combined_format.unwrap_or(false); + let options = options.map_or_else(Vec::new, |v| v.items.to_vec()); + let args = args.map_or_else(Vec::new, |v| v.items.to_vec()); + + let mut allowed_options = HashMap::::new(); + for opt in options { + let name = opt.name().to_string(); + if allowed_options + .insert(opt.name().to_string(), opt) + .is_some() + { + return Err(anyhow::format_err!("duplicate flag: {name}")); + } + } + + let program_spec = ProgramSpec::new( + program, + system_path, + option_bundling, + combined_format, + allowed_options, + args, + forbidden, + should_match + .map_or_else(Vec::new, |v| v.items.to_vec()) + .into_iter() + .map(|v| v.items.to_vec()) + .collect(), + should_not_match + .map_or_else(Vec::new, |v| v.items.to_vec()) + .into_iter() + .map(|v| v.items.to_vec()) + .collect(), + ); + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + policy_builder.add_program_spec(program_spec); + Ok(NoneType) + } + + fn forbid_substrings( + strings: UnpackList, + eval: &mut Evaluator, + ) -> anyhow::Result { + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + policy_builder.add_forbidden_substrings(&strings.items.to_vec()); + Ok(NoneType) + } + + fn forbid_program_regex( + regex: String, + reason: String, + eval: &mut Evaluator, + ) -> anyhow::Result { + let policy_builder = eval + .extra + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + let compiled_regex = regex::Regex::new(®ex)?; + policy_builder.add_forbidden_program_regex(compiled_regex, reason); + Ok(NoneType) + } + + fn opt(name: String, r#type: ArgMatcher, required: Option) -> anyhow::Result { + Ok(Opt::new( + name, + OptMeta::Value(r#type.arg_type()), + required.unwrap_or(false), + )) + } + + fn flag(name: String) -> anyhow::Result { + Ok(Opt::new(name, OptMeta::Flag, false)) + } +} diff --git a/codex-rs/execpolicy/src/program.rs b/codex-rs/execpolicy/src/program.rs new file mode 100644 index 0000000000..6984f5cb3c --- /dev/null +++ b/codex-rs/execpolicy/src/program.rs @@ -0,0 +1,247 @@ +use serde::Serialize; +use std::collections::HashMap; +use std::collections::HashSet; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_resolver::resolve_observed_args_with_patterns; +use crate::arg_resolver::PositionalArg; +use crate::error::Error; +use crate::error::Result; +use crate::opt::Opt; +use crate::opt::OptMeta; +use crate::valid_exec::MatchedFlag; +use crate::valid_exec::MatchedOpt; +use crate::valid_exec::ValidExec; +use crate::ArgType; +use crate::ExecCall; + +#[derive(Debug)] +pub struct ProgramSpec { + pub program: String, + pub system_path: Vec, + pub option_bundling: bool, + pub combined_format: bool, + pub allowed_options: HashMap, + pub arg_patterns: Vec, + forbidden: Option, + required_options: HashSet, + should_match: Vec>, + should_not_match: Vec>, +} + +impl ProgramSpec { + pub fn new( + program: String, + system_path: Vec, + option_bundling: bool, + combined_format: bool, + allowed_options: HashMap, + arg_patterns: Vec, + forbidden: Option, + should_match: Vec>, + should_not_match: Vec>, + ) -> Self { + let required_options = allowed_options + .iter() + .filter_map(|(name, opt)| { + if opt.required { + Some(name.clone()) + } else { + None + } + }) + .collect(); + Self { + program, + system_path, + option_bundling, + combined_format, + allowed_options, + arg_patterns, + forbidden, + required_options, + should_match, + should_not_match, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub enum MatchedExec { + Match { exec: ValidExec }, + Forbidden { cause: Forbidden, reason: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub enum Forbidden { + Program { + program: String, + exec_call: ExecCall, + }, + Arg { + arg: String, + exec_call: ExecCall, + }, + Exec { + exec: ValidExec, + }, +} + +impl ProgramSpec { + // TODO(mbolin): The idea is that there should be a set of rules defined for + // a program and the args should be checked against the rules to determine + // if the program should be allowed to run. + pub fn check(&self, exec_call: &ExecCall) -> Result { + let mut expecting_option_value: Option<(String, ArgType)> = None; + let mut args = Vec::::new(); + let mut matched_flags = Vec::::new(); + let mut matched_opts = Vec::::new(); + + for (index, arg) in exec_call.args.iter().enumerate() { + if let Some(expected) = expecting_option_value { + // If we are expecting an option value, then the next argument + // should be the value for the option. + // This had better not be another option! + let (name, arg_type) = expected; + if arg.starts_with("-") { + return Err(Error::OptionFollowedByOptionInsteadOfValue { + program: self.program.clone(), + option: name, + value: arg.clone(), + }); + } + + matched_opts.push(MatchedOpt::new(&name, arg, arg_type)?); + expecting_option_value = None; + } else if arg == "--" { + return Err(Error::DoubleDashNotSupportedYet { + program: self.program.clone(), + }); + } else if arg.starts_with("-") { + match self.allowed_options.get(arg) { + Some(opt) => { + match &opt.meta { + OptMeta::Flag => { + matched_flags.push(MatchedFlag { name: arg.clone() }); + // A flag does not expect an argument: continue. + continue; + } + OptMeta::Value(arg_type) => { + expecting_option_value = Some((arg.clone(), arg_type.clone())); + continue; + } + } + } + None => { + // It could be an --option=value style flag... + } + } + + return Err(Error::UnknownOption { + program: self.program.clone(), + option: arg.clone(), + }); + } else { + args.push(PositionalArg { + index, + value: arg.clone(), + }); + } + } + + if let Some(expected) = expecting_option_value { + let (name, _arg_type) = expected; + return Err(Error::OptionMissingValue { + program: self.program.clone(), + option: name, + }); + } + + let matched_args = + resolve_observed_args_with_patterns(&self.program, args, &self.arg_patterns)?; + + // Verify all required options are present. + let matched_opt_names: HashSet = matched_opts + .iter() + .map(|opt| opt.name().to_string()) + .collect(); + if !matched_opt_names.is_superset(&self.required_options) { + let mut options = self + .required_options + .difference(&matched_opt_names) + .map(|s| s.to_string()) + .collect::>(); + options.sort(); + return Err(Error::MissingRequiredOptions { + program: self.program.clone(), + options, + }); + } + + let exec = ValidExec { + program: self.program.clone(), + flags: matched_flags, + opts: matched_opts, + args: matched_args, + system_path: self.system_path.clone(), + }; + match &self.forbidden { + Some(reason) => Ok(MatchedExec::Forbidden { + cause: Forbidden::Exec { exec }, + reason: reason.clone(), + }), + None => Ok(MatchedExec::Match { exec }), + } + } + + pub fn verify_should_match_list(&self) -> Vec { + let mut violations = Vec::new(); + for good in &self.should_match { + let exec_call = ExecCall { + program: self.program.clone(), + args: good.clone(), + }; + match self.check(&exec_call) { + Ok(_) => {} + Err(error) => { + violations.push(PositiveExampleFailedCheck { + program: self.program.clone(), + args: good.clone(), + error, + }); + } + } + } + violations + } + + pub fn verify_should_not_match_list(&self) -> Vec { + let mut violations = Vec::new(); + for bad in &self.should_not_match { + let exec_call = ExecCall { + program: self.program.clone(), + args: bad.clone(), + }; + if self.check(&exec_call).is_ok() { + violations.push(NegativeExamplePassedCheck { + program: self.program.clone(), + args: bad.clone(), + }); + } + } + violations + } +} + +#[derive(Debug, Eq, PartialEq)] +pub struct PositiveExampleFailedCheck { + pub program: String, + pub args: Vec, + pub error: Error, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct NegativeExamplePassedCheck { + pub program: String, + pub args: Vec, +} diff --git a/codex-rs/execpolicy/src/sed_command.rs b/codex-rs/execpolicy/src/sed_command.rs new file mode 100644 index 0000000000..64494ddf00 --- /dev/null +++ b/codex-rs/execpolicy/src/sed_command.rs @@ -0,0 +1,17 @@ +use crate::error::Error; +use crate::error::Result; + +pub fn parse_sed_command(sed_command: &str) -> Result<()> { + // For now, we parse only commands like `122,202p`. + if let Some(stripped) = sed_command.strip_suffix("p") { + if let Some((first, rest)) = stripped.split_once(",") { + if first.parse::().is_ok() && rest.parse::().is_ok() { + return Ok(()); + } + } + } + + Err(Error::SedCommandNotProvablySafe { + command: sed_command.to_string(), + }) +} diff --git a/codex-rs/execpolicy/src/valid_exec.rs b/codex-rs/execpolicy/src/valid_exec.rs new file mode 100644 index 0000000000..0cc3b239ca --- /dev/null +++ b/codex-rs/execpolicy/src/valid_exec.rs @@ -0,0 +1,95 @@ +use crate::arg_type::ArgType; +use crate::error::Result; +use serde::Serialize; + +/// exec() invocation that has been accepted by a `Policy`. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct ValidExec { + pub program: String, + pub flags: Vec, + pub opts: Vec, + pub args: Vec, + + /// If non-empty, a prioritized list of paths to try instead of `program`. + /// For example, `/bin/ls` is harder to compromise than whatever `ls` + /// happens to be in the user's `$PATH`, so `/bin/ls` would be included for + /// `ls`. The caller is free to disregard this list and use `program`. + pub system_path: Vec, +} + +impl ValidExec { + pub fn new(program: &str, args: Vec, system_path: &[&str]) -> Self { + Self { + program: program.to_string(), + flags: vec![], + opts: vec![], + args, + system_path: system_path.iter().map(|&s| s.to_string()).collect(), + } + } + + /// Whether a possible side effect of running this command includes writing + /// a file. + pub fn might_write_files(&self) -> bool { + self.opts.iter().any(|opt| opt.r#type.might_write_file()) + || self.args.iter().any(|opt| opt.r#type.might_write_file()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedArg { + pub index: usize, + pub r#type: ArgType, + pub value: String, +} + +impl MatchedArg { + pub fn new(index: usize, r#type: ArgType, value: &str) -> Result { + r#type.validate(value)?; + Ok(Self { + index, + r#type, + value: value.to_string(), + }) + } +} + +/// A match for an option declared with opt() in a .policy file. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedOpt { + /// Name of the option that was matched. + pub name: String, + /// Value supplied for the option. + pub value: String, + /// Type of the value supplied for the option. + pub r#type: ArgType, +} + +impl MatchedOpt { + pub fn new(name: &str, value: &str, r#type: ArgType) -> Result { + r#type.validate(value)?; + Ok(Self { + name: name.to_string(), + value: value.to_string(), + r#type, + }) + } + + pub fn name(&self) -> &str { + &self.name + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct MatchedFlag { + /// Name of the flag that was matched. + pub name: String, +} + +impl MatchedFlag { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + } + } +} diff --git a/codex-rs/execpolicy/tests/bad.rs b/codex-rs/execpolicy/tests/bad.rs new file mode 100644 index 0000000000..91f8b52ba4 --- /dev/null +++ b/codex-rs/execpolicy/tests/bad.rs @@ -0,0 +1,9 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::NegativeExamplePassedCheck; + +#[test] +fn verify_everything_in_bad_list_is_rejected() { + let policy = get_default_policy().expect("failed to load default policy"); + let violations = policy.check_each_bad_list_individually(); + assert_eq!(Vec::::new(), violations); +} diff --git a/codex-rs/execpolicy/tests/cp.rs b/codex-rs/execpolicy/tests/cp.rs new file mode 100644 index 0000000000..8981ac7a34 --- /dev/null +++ b/codex-rs/execpolicy/tests/cp.rs @@ -0,0 +1,85 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgMatcher; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_cp_no_args() { + let policy = setup(); + let cp = ExecCall::new("cp", &[]); + assert_eq!( + Err(Error::NotEnoughArgs { + program: "cp".to_string(), + args: vec![], + arg_patterns: vec![ArgMatcher::ReadableFiles, ArgMatcher::WriteableFile] + }), + policy.check(&cp) + ) +} + +#[test] +fn test_cp_one_arg() { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo/bar"]); + + assert_eq!( + Err(Error::VarargMatcherDidNotMatchAnything { + program: "cp".to_string(), + matcher: ArgMatcher::ReadableFiles, + }), + policy.check(&cp) + ); +} + +#[test] +fn test_cp_one_file() -> Result<()> { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo/bar", "../baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "cp", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo/bar")?, + MatchedArg::new(1, ArgType::WriteableFile, "../baz")?, + ], + &["/bin/cp", "/usr/bin/cp"] + ) + }), + policy.check(&cp) + ); + Ok(()) +} + +#[test] +fn test_cp_multiple_files() -> Result<()> { + let policy = setup(); + let cp = ExecCall::new("cp", &["foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "cp", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo")?, + MatchedArg::new(1, ArgType::ReadableFile, "bar")?, + MatchedArg::new(2, ArgType::WriteableFile, "baz")?, + ], + &["/bin/cp", "/usr/bin/cp"] + ) + }), + policy.check(&cp) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/good.rs b/codex-rs/execpolicy/tests/good.rs new file mode 100644 index 0000000000..18a002850c --- /dev/null +++ b/codex-rs/execpolicy/tests/good.rs @@ -0,0 +1,9 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::PositiveExampleFailedCheck; + +#[test] +fn verify_everything_in_good_list_is_allowed() { + let policy = get_default_policy().expect("failed to load default policy"); + let violations = policy.check_each_good_list_individually(); + assert_eq!(Vec::::new(), violations); +} diff --git a/codex-rs/execpolicy/tests/head.rs b/codex-rs/execpolicy/tests/head.rs new file mode 100644 index 0000000000..196de081f6 --- /dev/null +++ b/codex-rs/execpolicy/tests/head.rs @@ -0,0 +1,132 @@ +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgMatcher; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedOpt; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +extern crate codex_execpolicy; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_head_no_args() { + let policy = setup(); + let head = ExecCall::new("head", &[]); + // It is actually valid to call `head` without arguments: it will read from + // stdin instead of from a file. Though recall that a command rejected by + // the policy is not "unsafe:" it just means that this library cannot + // *guarantee* that the command is safe. + // + // If we start verifying individual components of a shell command, such as: + // `find . -name | head -n 10`, then it might be important to allow the + // no-arg case. + assert_eq!( + Err(Error::VarargMatcherDidNotMatchAnything { + program: "head".to_string(), + matcher: ArgMatcher::ReadableFiles, + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_one_file_no_flags() -> Result<()> { + let policy = setup(); + let head = ExecCall::new("head", &["src/extension.ts"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "head", + vec![MatchedArg::new( + 0, + ArgType::ReadableFile, + "src/extension.ts" + )?], + &["/bin/head", "/usr/bin/head"] + ) + }), + policy.check(&head) + ); + Ok(()) +} + +#[test] +fn test_head_one_flag_one_file() -> Result<()> { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "100", "src/extension.ts"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "head".to_string(), + flags: vec![], + opts: vec![MatchedOpt::new("-n", "100", ArgType::PositiveInteger).unwrap()], + args: vec![MatchedArg::new( + 2, + ArgType::ReadableFile, + "src/extension.ts" + )?], + system_path: vec!["/bin/head".to_string(), "/usr/bin/head".to_string()], + } + }), + policy.check(&head) + ); + Ok(()) +} + +#[test] +fn test_head_invalid_n_as_0() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "0", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "0".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_nonint_float() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "1.5", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "1.5".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_float() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "1.0", "src/extension.ts"]); + assert_eq!( + Err(Error::InvalidPositiveInteger { + value: "1.0".to_string(), + }), + policy.check(&head) + ) +} + +#[test] +fn test_head_invalid_n_as_negative_int() { + let policy = setup(); + let head = ExecCall::new("head", &["-n", "-1", "src/extension.ts"]); + assert_eq!( + Err(Error::OptionFollowedByOptionInsteadOfValue { + program: "head".to_string(), + option: "-n".to_string(), + value: "-1".to_string(), + }), + policy.check(&head) + ) +} diff --git a/codex-rs/execpolicy/tests/literal.rs b/codex-rs/execpolicy/tests/literal.rs new file mode 100644 index 0000000000..d849371e3b --- /dev/null +++ b/codex-rs/execpolicy/tests/literal.rs @@ -0,0 +1,50 @@ +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::PolicyParser; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +extern crate codex_execpolicy; + +#[test] +fn test_invalid_subcommand() -> Result<()> { + let unparsed_policy = r#" +define_program( + program="fake_executable", + args=["subcommand", "sub-subcommand"], +) +"#; + let parser = PolicyParser::new("test_invalid_subcommand", unparsed_policy); + let policy = parser.parse().expect("failed to parse policy"); + let valid_call = ExecCall::new("fake_executable", &["subcommand", "sub-subcommand"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "fake_executable", + vec![ + MatchedArg::new(0, ArgType::Literal("subcommand".to_string()), "subcommand")?, + MatchedArg::new( + 1, + ArgType::Literal("sub-subcommand".to_string()), + "sub-subcommand" + )?, + ], + &[] + ) + }), + policy.check(&valid_call) + ); + + let invalid_call = ExecCall::new("fake_executable", &["subcommand", "not-a-real-subcommand"]); + assert_eq!( + Err(Error::LiteralValueDidNotMatch { + expected: "sub-subcommand".to_string(), + actual: "not-a-real-subcommand".to_string() + }), + policy.check(&invalid_call) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/ls.rs b/codex-rs/execpolicy/tests/ls.rs new file mode 100644 index 0000000000..f7e78f22f3 --- /dev/null +++ b/codex-rs/execpolicy/tests/ls.rs @@ -0,0 +1,166 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_ls_no_args() { + let policy = setup(); + let ls = ExecCall::new("ls", &[]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new("ls", vec![], &["/bin/ls", "/usr/bin/ls"]) + }), + policy.check(&ls) + ); +} + +#[test] +fn test_ls_dash_a_dash_l() { + let policy = setup(); + let args = &["-a", "-l"]; + let ls_a_l = ExecCall::new("ls", args); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-a"), MatchedFlag::new("-l")], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_a_l) + ); +} + +#[test] +fn test_ls_dash_z() { + let policy = setup(); + + // -z is currently an invalid option for ls, but it has so many options, + // perhaps it will get added at some point... + let ls_z = ExecCall::new("ls", &["-z"]); + assert_eq!( + Err(Error::UnknownOption { + program: "ls".into(), + option: "-z".into() + }), + policy.check(&ls_z) + ); +} + +#[test] +fn test_ls_dash_al() { + let policy = setup(); + + // This currently fails, but it should pass once option_bundling=True is implemented. + let ls_al = ExecCall::new("ls", &["-al"]); + assert_eq!( + Err(Error::UnknownOption { + program: "ls".into(), + option: "-al".into() + }), + policy.check(&ls_al) + ); +} + +#[test] +fn test_ls_one_file_arg() -> Result<()> { + let policy = setup(); + + let ls_one_file_arg = ExecCall::new("ls", &["foo"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "ls", + vec![MatchedArg::new(0, ArgType::ReadableFile, "foo")?], + &["/bin/ls", "/usr/bin/ls"] + ) + }), + policy.check(&ls_one_file_arg) + ); + Ok(()) +} + +#[test] +fn test_ls_multiple_file_args() -> Result<()> { + let policy = setup(); + + let ls_multiple_file_args = ExecCall::new("ls", &["foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec::new( + "ls", + vec![ + MatchedArg::new(0, ArgType::ReadableFile, "foo")?, + MatchedArg::new(1, ArgType::ReadableFile, "bar")?, + MatchedArg::new(2, ArgType::ReadableFile, "baz")?, + ], + &["/bin/ls", "/usr/bin/ls"] + ) + }), + policy.check(&ls_multiple_file_args) + ); + Ok(()) +} + +#[test] +fn test_ls_multiple_flags_and_file_args() -> Result<()> { + let policy = setup(); + + let ls_multiple_flags_and_file_args = ExecCall::new("ls", &["-l", "-a", "foo", "bar", "baz"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-l"), MatchedFlag::new("-a")], + args: vec![ + MatchedArg::new(2, ArgType::ReadableFile, "foo")?, + MatchedArg::new(3, ArgType::ReadableFile, "bar")?, + MatchedArg::new(4, ArgType::ReadableFile, "baz")?, + ], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_multiple_flags_and_file_args) + ); + Ok(()) +} + +#[test] +fn test_flags_after_file_args() -> Result<()> { + let policy = setup(); + + // TODO(mbolin): While this is "safe" in that it will not do anything bad + // to the user's machine, it will fail because apparently `ls` does not + // allow flags after file arguments (as some commands do). We should + // extend define_program() to make this part of the configuration so that + // this command is disallowed. + let ls_flags_after_file_args = ExecCall::new("ls", &["foo", "-l"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "ls".into(), + flags: vec![MatchedFlag::new("-l")], + args: vec![MatchedArg::new(0, ArgType::ReadableFile, "foo")?], + system_path: ["/bin/ls".into(), "/usr/bin/ls".into()].into(), + ..Default::default() + } + }), + policy.check(&ls_flags_after_file_args) + ); + Ok(()) +} diff --git a/codex-rs/execpolicy/tests/parse_sed_command.rs b/codex-rs/execpolicy/tests/parse_sed_command.rs new file mode 100644 index 0000000000..6d03b626ef --- /dev/null +++ b/codex-rs/execpolicy/tests/parse_sed_command.rs @@ -0,0 +1,23 @@ +use codex_execpolicy::parse_sed_command; +use codex_execpolicy::Error; + +#[test] +fn parses_simple_print_command() { + assert_eq!(parse_sed_command("122,202p"), Ok(())); +} + +#[test] +fn rejects_malformed_print_command() { + assert_eq!( + parse_sed_command("122,202"), + Err(Error::SedCommandNotProvablySafe { + command: "122,202".to_string(), + }) + ); + assert_eq!( + parse_sed_command("122202"), + Err(Error::SedCommandNotProvablySafe { + command: "122202".to_string(), + }) + ); +} diff --git a/codex-rs/execpolicy/tests/pwd.rs b/codex-rs/execpolicy/tests/pwd.rs new file mode 100644 index 0000000000..4e29e4cbc1 --- /dev/null +++ b/codex-rs/execpolicy/tests/pwd.rs @@ -0,0 +1,85 @@ +extern crate codex_execpolicy; + +use std::vec; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::Policy; +use codex_execpolicy::PositionalArg; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_pwd_no_args() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &[]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_capital_l() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["-L"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + flags: vec![MatchedFlag::new("-L")], + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_capital_p() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["-P"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "pwd".into(), + flags: vec![MatchedFlag::new("-P")], + ..Default::default() + } + }), + policy.check(&pwd) + ); +} + +#[test] +fn test_pwd_extra_args() { + let policy = setup(); + let pwd = ExecCall::new("pwd", &["foo", "bar"]); + assert_eq!( + Err(Error::UnexpectedArguments { + program: "pwd".to_string(), + args: vec![ + PositionalArg { + index: 0, + value: "foo".to_string() + }, + PositionalArg { + index: 1, + value: "bar".to_string() + }, + ], + }), + policy.check(&pwd) + ); +} diff --git a/codex-rs/execpolicy/tests/sed.rs b/codex-rs/execpolicy/tests/sed.rs new file mode 100644 index 0000000000..cc26bf1eb4 --- /dev/null +++ b/codex-rs/execpolicy/tests/sed.rs @@ -0,0 +1,83 @@ +extern crate codex_execpolicy; + +use codex_execpolicy::get_default_policy; +use codex_execpolicy::ArgType; +use codex_execpolicy::Error; +use codex_execpolicy::ExecCall; +use codex_execpolicy::MatchedArg; +use codex_execpolicy::MatchedExec; +use codex_execpolicy::MatchedFlag; +use codex_execpolicy::MatchedOpt; +use codex_execpolicy::Policy; +use codex_execpolicy::Result; +use codex_execpolicy::ValidExec; + +fn setup() -> Policy { + get_default_policy().expect("failed to load default policy") +} + +#[test] +fn test_sed_print_specific_lines() -> Result<()> { + let policy = setup(); + let sed = ExecCall::new("sed", &["-n", "122,202p", "hello.txt"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "sed".to_string(), + flags: vec![MatchedFlag::new("-n")], + args: vec![ + MatchedArg::new(1, ArgType::SedCommand, "122,202p")?, + MatchedArg::new(2, ArgType::ReadableFile, "hello.txt")?, + ], + system_path: vec!["/usr/bin/sed".to_string()], + ..Default::default() + } + }), + policy.check(&sed) + ); + Ok(()) +} + +#[test] +fn test_sed_print_specific_lines_with_e_flag() -> Result<()> { + let policy = setup(); + let sed = ExecCall::new("sed", &["-n", "-e", "122,202p", "hello.txt"]); + assert_eq!( + Ok(MatchedExec::Match { + exec: ValidExec { + program: "sed".to_string(), + flags: vec![MatchedFlag::new("-n")], + opts: vec![MatchedOpt::new("-e", "122,202p", ArgType::SedCommand).unwrap()], + args: vec![MatchedArg::new(3, ArgType::ReadableFile, "hello.txt")?], + system_path: vec!["/usr/bin/sed".to_string()], + } + }), + policy.check(&sed) + ); + Ok(()) +} + +#[test] +fn test_sed_reject_dangerous_command() { + let policy = setup(); + let sed = ExecCall::new("sed", &["-e", "s/y/echo hi/e", "hello.txt"]); + assert_eq!( + Err(Error::SedCommandNotProvablySafe { + command: "s/y/echo hi/e".to_string(), + }), + policy.check(&sed) + ); +} + +#[test] +fn test_sed_verify_e_or_pattern_is_required() { + let policy = setup(); + let sed = ExecCall::new("sed", &["122,202p"]); + assert_eq!( + Err(Error::MissingRequiredOptions { + program: "sed".to_string(), + options: vec!["-e".to_string()], + }), + policy.check(&sed) + ); +} From d2cb604b7dd9ce0a921f4283462324a5591f453f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 17:46:10 -0700 Subject: [PATCH 57/84] fix: close stdin when running an exec tool call --- codex-rs/core/src/exec.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index fe6bad548e..ae83dc84e7 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -206,9 +206,17 @@ pub async fn exec( if let Some(dir) = &workdir { cmd.current_dir(dir); } - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - cmd.kill_on_drop(true); - cmd.spawn()? + + // Do not create a file descriptor for stdin because otherwise some + // commands may hang forever waiting for input. For example, ripgrep has + // a heuristic where it may try to read from stdin as explained here: + // https://github.com/BurntSushi/ripgrep/blob/e2362d4d5185d02fa857bf381e7bd52e66fafc73/crates/core/flags/hiargs.rs#L1101-L1103 + cmd.stdin(Stdio::null()); + + cmd.stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn()? }; let stdout_handle = tokio::spawn(read_capped( From c9dffbaf042a0c738930cc8759f5bf23774a3369 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 17:54:42 -0700 Subject: [PATCH 58/84] fix: for now, only run rust-ci.yml on PRs that modify files in codex-rs --- .github/workflows/rust-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 1867949d53..851650e629 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -1,7 +1,13 @@ name: rust-ci on: - pull_request: { branches: [main] } - push: { branches: [main] } + pull_request: + branches: + - main + paths: + - 'codex-rs/**' + push: + branches: + - main # For CI, we build in debug (`--profile dev`) rather than release mode so we # get signal faster. From eec2b1a3d4d3f4b89acee70d30cd3f198ce22cb6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 17:54:42 -0700 Subject: [PATCH 59/84] fix: for now, only run rust-ci.yml on PRs that modify files in codex-rs --- .github/workflows/rust-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 1867949d53..d4efca0e28 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -1,7 +1,13 @@ name: rust-ci on: - pull_request: { branches: [main] } - push: { branches: [main] } + pull_request: + branches: + - main + paths: + - "codex-rs/**" + push: + branches: + - main # For CI, we build in debug (`--profile dev`) rather than release mode so we # get signal faster. From 4db909caa7a2be6700dff27bc9c6ad098f48c270 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 18:03:21 -0700 Subject: [PATCH 60/84] fix: add RUST_BACKTRACE=full when running `cargo test` in CI --- .github/workflows/rust-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index d4efca0e28..0bc3ee0ccd 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -32,7 +32,7 @@ jobs: run: cargo fmt -- --config imports_granularity=Item --check || echo "FAILED=${FAILED:+$FAILED, }cargo fmt" >> $GITHUB_ENV - name: cargo test - run: cargo test || echo "FAILED=${FAILED:+$FAILED, }cargo test" >> $GITHUB_ENV + run: RUST_BACKTRACE=full cargo test || echo "FAILED=${FAILED:+$FAILED, }cargo test" >> $GITHUB_ENV - name: cargo clippy run: cargo clippy --all-features -- -D warnings || echo "FAILED=${FAILED:+$FAILED, }cargo clippy" >> $GITHUB_ENV @@ -75,7 +75,7 @@ jobs: run: cargo fmt -- --config imports_granularity=Item --check || echo "FAILED=${FAILED:+$FAILED, }cargo fmt" >> $GITHUB_ENV - name: cargo test - run: cargo test || echo "FAILED=${FAILED:+$FAILED, }cargo test" >> $GITHUB_ENV + run: RUST_BACKTRACE=full cargo test || echo "FAILED=${FAILED:+$FAILED, }cargo test" >> $GITHUB_ENV - name: cargo clippy run: cargo clippy --all-features -- -D warnings || echo "FAILED=${FAILED:+$FAILED, }cargo clippy" >> $GITHUB_ENV From ea4557f9631e7829c84fd4205135fc14cb99d596 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 20:10:52 -0700 Subject: [PATCH 61/84] feat(tui-rs): add support for mousewheel scrolling --- codex-rs/tui/src/app.rs | 34 ++++++++++++++++--- codex-rs/tui/src/app_event.rs | 4 +++ codex-rs/tui/src/chatwidget.rs | 17 ++++++++++ .../tui/src/conversation_history_widget.rs | 31 +++++++++++------ codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/tui.rs | 4 +++ 6 files changed, 77 insertions(+), 14 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..26a7074b5b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2,6 +2,7 @@ use crate::app_event::AppEvent; use crate::chatwidget::ChatWidget; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; +use crate::scroll_event_helper::ScrollEventHelper; use crate::tui; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; @@ -10,6 +11,8 @@ use codex_core::protocol::SandboxPolicy; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; use std::sync::mpsc::Sender; @@ -39,6 +42,7 @@ impl App<'_> { model: Option, ) -> Self { let (app_event_tx, app_event_rx) = channel(); + let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -49,10 +53,21 @@ impl App<'_> { let app_event = match event { crossterm::event::Event::Key(key_event) => AppEvent::KeyEvent(key_event), crossterm::event::Event::Resize(_, _) => AppEvent::Redraw, - crossterm::event::Event::FocusGained - | crossterm::event::Event::FocusLost - | crossterm::event::Event::Mouse(_) - | crossterm::event::Event::Paste(_) => { + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollUp, + .. + }) => { + scroll_event_helper.scroll_up(); + continue; + } + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollDown, + .. + }) => { + scroll_event_helper.scroll_down(); + continue; + } + _ => { continue; } }; @@ -125,6 +140,9 @@ impl App<'_> { } }; } + AppEvent::Scroll(scroll_delta) => { + self.dispatch_scroll_event(scroll_delta); + } AppEvent::CodexEvent(event) => { self.dispatch_codex_event(event); } @@ -184,6 +202,14 @@ impl App<'_> { } } + fn dispatch_scroll_event(&mut self, scroll_delta: i32) { + if matches!(self.app_state, AppState::Chat) { + if let Err(e) = self.chat_widget.handle_scroll_delta(scroll_delta) { + tracing::error!("SendError: {e}"); + } + } + } + fn dispatch_codex_event(&mut self, event: Event) { if matches!(self.app_state, AppState::Chat) { if let Err(e) = self.chat_widget.handle_codex_event(event) { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index bb8efb8e15..932ad9c65c 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -3,7 +3,11 @@ use crossterm::event::KeyEvent; pub(crate) enum AppEvent { CodexEvent(Event), + + Scroll(i32), + Redraw, + KeyEvent(KeyEvent), /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..64cb896d93 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -364,6 +364,23 @@ impl ChatWidget<'_> { Ok(()) } + pub(crate) fn handle_scroll_delta( + &mut self, + scroll_delta: i32, + ) -> std::result::Result<(), std::sync::mpsc::SendError> { + // If the user is trying to scroll exactly one line, we let them, but + // otherwise we assume they are trying to scroll in larger increments. + let magnified_scroll_delta = if scroll_delta == 1 { + 1 + } else { + // Play with this: perhaps it should be non-linear? + scroll_delta * 2 + }; + self.conversation_history.scroll(magnified_scroll_delta); + self.request_redraw()?; + Ok(()) + } + /// Forward an `Op` directly to codex. pub(crate) fn submit_op(&self, op: Op) { if let Err(e) = self.codex_op_tx.send(op) { diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index c8f6906169..27b5e9b3cf 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -40,11 +40,11 @@ impl ConversationHistoryWidget { pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) -> bool { match key_event.code { KeyCode::Up | KeyCode::Char('k') => { - self.scroll_up(); + self.scroll_up(1); true } KeyCode::Down | KeyCode::Char('j') => { - self.scroll_down(); + self.scroll_down(1); true } KeyCode::PageUp | KeyCode::Char('b') | KeyCode::Char('u') | KeyCode::Char('U') => { @@ -59,9 +59,18 @@ impl ConversationHistoryWidget { } } - fn scroll_up(&mut self) { - // If a user is scrolling up from the "stick to bottom" mode, we - // need to scroll them back such that they move just one line up. + /// Negative delta scrolls up; positive delta scrolls down. + pub(crate) fn scroll(&mut self, delta: i32) { + match delta.cmp(&0) { + std::cmp::Ordering::Less => self.scroll_up(-delta as u32), + std::cmp::Ordering::Greater => self.scroll_down(delta as u32), + std::cmp::Ordering::Equal => {} + } + } + + fn scroll_up(&mut self, num_lines: u32) { + // If a user is scrolling up from the "stick to bottom" mode, we need to + // map this to a specific scroll position so we can caluate the delta. // This requires us to care about how tall the screen is. if self.scroll_position == usize::MAX { self.scroll_position = self @@ -70,24 +79,26 @@ impl ConversationHistoryWidget { .saturating_sub(self.last_viewport_height.get()); } - self.scroll_position = self.scroll_position.saturating_sub(1); + self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); } - fn scroll_down(&mut self) { + fn scroll_down(&mut self, num_lines: u32) { // If we're already pinned to the bottom there's nothing to do. if self.scroll_position == usize::MAX { return; } let viewport_height = self.last_viewport_height.get().max(1); - let num_lines = self.num_rendered_lines.get(); + let num_rendered_lines = self.num_rendered_lines.get(); // Compute the maximum explicit scroll offset that still shows a full // viewport. This mirrors the calculation in `scroll_page_down()` and // in the render path. - let max_scroll = num_lines.saturating_sub(viewport_height).saturating_add(1); + let max_scroll = num_rendered_lines + .saturating_sub(viewport_height) + .saturating_add(1); - let new_pos = self.scroll_position.saturating_add(1); + let new_pos = self.scroll_position.saturating_add(num_lines as usize); if new_pos >= max_scroll { // Reached (or passed) the bottom – switch to stick‑to‑bottom mode diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..7361663bb4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -21,6 +21,7 @@ mod exec_command; mod git_warning_screen; mod history_cell; mod log_layer; +mod scroll_event_helper; mod status_indicator_widget; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 0753dcb07a..8cc54460a4 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -2,6 +2,8 @@ use std::io::stdout; use std::io::Stdout; use std::io::{self}; +use crossterm::event::DisableMouseCapture; +use crossterm::event::EnableMouseCapture; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -16,6 +18,7 @@ pub type Tui = Terminal>; /// Initialize the terminal pub fn init() -> io::Result { execute!(stdout(), EnterAlternateScreen)?; + execute!(stdout(), EnableMouseCapture)?; enable_raw_mode()?; set_panic_hook(); Terminal::new(CrosstermBackend::new(stdout())) @@ -31,6 +34,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> io::Result<()> { + execute!(stdout(), DisableMouseCapture)?; execute!(stdout(), LeaveAlternateScreen)?; disable_raw_mode()?; Ok(()) From 006da34ada673899e116de387c037d5f843fffb6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 20:10:52 -0700 Subject: [PATCH 62/84] feat(tui-rs): add support for mousewheel scrolling --- codex-rs/tui/src/app.rs | 34 +++++++-- codex-rs/tui/src/app_event.rs | 4 ++ codex-rs/tui/src/chatwidget.rs | 17 +++++ .../tui/src/conversation_history_widget.rs | 31 +++++--- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/scroll_event_helper.rs | 71 +++++++++++++++++++ codex-rs/tui/src/tui.rs | 4 ++ 7 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 codex-rs/tui/src/scroll_event_helper.rs diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..26a7074b5b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2,6 +2,7 @@ use crate::app_event::AppEvent; use crate::chatwidget::ChatWidget; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; +use crate::scroll_event_helper::ScrollEventHelper; use crate::tui; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; @@ -10,6 +11,8 @@ use codex_core::protocol::SandboxPolicy; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; use std::sync::mpsc::Sender; @@ -39,6 +42,7 @@ impl App<'_> { model: Option, ) -> Self { let (app_event_tx, app_event_rx) = channel(); + let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -49,10 +53,21 @@ impl App<'_> { let app_event = match event { crossterm::event::Event::Key(key_event) => AppEvent::KeyEvent(key_event), crossterm::event::Event::Resize(_, _) => AppEvent::Redraw, - crossterm::event::Event::FocusGained - | crossterm::event::Event::FocusLost - | crossterm::event::Event::Mouse(_) - | crossterm::event::Event::Paste(_) => { + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollUp, + .. + }) => { + scroll_event_helper.scroll_up(); + continue; + } + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollDown, + .. + }) => { + scroll_event_helper.scroll_down(); + continue; + } + _ => { continue; } }; @@ -125,6 +140,9 @@ impl App<'_> { } }; } + AppEvent::Scroll(scroll_delta) => { + self.dispatch_scroll_event(scroll_delta); + } AppEvent::CodexEvent(event) => { self.dispatch_codex_event(event); } @@ -184,6 +202,14 @@ impl App<'_> { } } + fn dispatch_scroll_event(&mut self, scroll_delta: i32) { + if matches!(self.app_state, AppState::Chat) { + if let Err(e) = self.chat_widget.handle_scroll_delta(scroll_delta) { + tracing::error!("SendError: {e}"); + } + } + } + fn dispatch_codex_event(&mut self, event: Event) { if matches!(self.app_state, AppState::Chat) { if let Err(e) = self.chat_widget.handle_codex_event(event) { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index bb8efb8e15..932ad9c65c 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -3,7 +3,11 @@ use crossterm::event::KeyEvent; pub(crate) enum AppEvent { CodexEvent(Event), + + Scroll(i32), + Redraw, + KeyEvent(KeyEvent), /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..64cb896d93 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -364,6 +364,23 @@ impl ChatWidget<'_> { Ok(()) } + pub(crate) fn handle_scroll_delta( + &mut self, + scroll_delta: i32, + ) -> std::result::Result<(), std::sync::mpsc::SendError> { + // If the user is trying to scroll exactly one line, we let them, but + // otherwise we assume they are trying to scroll in larger increments. + let magnified_scroll_delta = if scroll_delta == 1 { + 1 + } else { + // Play with this: perhaps it should be non-linear? + scroll_delta * 2 + }; + self.conversation_history.scroll(magnified_scroll_delta); + self.request_redraw()?; + Ok(()) + } + /// Forward an `Op` directly to codex. pub(crate) fn submit_op(&self, op: Op) { if let Err(e) = self.codex_op_tx.send(op) { diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index c8f6906169..27b5e9b3cf 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -40,11 +40,11 @@ impl ConversationHistoryWidget { pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) -> bool { match key_event.code { KeyCode::Up | KeyCode::Char('k') => { - self.scroll_up(); + self.scroll_up(1); true } KeyCode::Down | KeyCode::Char('j') => { - self.scroll_down(); + self.scroll_down(1); true } KeyCode::PageUp | KeyCode::Char('b') | KeyCode::Char('u') | KeyCode::Char('U') => { @@ -59,9 +59,18 @@ impl ConversationHistoryWidget { } } - fn scroll_up(&mut self) { - // If a user is scrolling up from the "stick to bottom" mode, we - // need to scroll them back such that they move just one line up. + /// Negative delta scrolls up; positive delta scrolls down. + pub(crate) fn scroll(&mut self, delta: i32) { + match delta.cmp(&0) { + std::cmp::Ordering::Less => self.scroll_up(-delta as u32), + std::cmp::Ordering::Greater => self.scroll_down(delta as u32), + std::cmp::Ordering::Equal => {} + } + } + + fn scroll_up(&mut self, num_lines: u32) { + // If a user is scrolling up from the "stick to bottom" mode, we need to + // map this to a specific scroll position so we can caluate the delta. // This requires us to care about how tall the screen is. if self.scroll_position == usize::MAX { self.scroll_position = self @@ -70,24 +79,26 @@ impl ConversationHistoryWidget { .saturating_sub(self.last_viewport_height.get()); } - self.scroll_position = self.scroll_position.saturating_sub(1); + self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); } - fn scroll_down(&mut self) { + fn scroll_down(&mut self, num_lines: u32) { // If we're already pinned to the bottom there's nothing to do. if self.scroll_position == usize::MAX { return; } let viewport_height = self.last_viewport_height.get().max(1); - let num_lines = self.num_rendered_lines.get(); + let num_rendered_lines = self.num_rendered_lines.get(); // Compute the maximum explicit scroll offset that still shows a full // viewport. This mirrors the calculation in `scroll_page_down()` and // in the render path. - let max_scroll = num_lines.saturating_sub(viewport_height).saturating_add(1); + let max_scroll = num_rendered_lines + .saturating_sub(viewport_height) + .saturating_add(1); - let new_pos = self.scroll_position.saturating_add(1); + let new_pos = self.scroll_position.saturating_add(num_lines as usize); if new_pos >= max_scroll { // Reached (or passed) the bottom – switch to stick‑to‑bottom mode diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..7361663bb4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -21,6 +21,7 @@ mod exec_command; mod git_warning_screen; mod history_cell; mod log_layer; +mod scroll_event_helper; mod status_indicator_widget; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/scroll_event_helper.rs b/codex-rs/tui/src/scroll_event_helper.rs new file mode 100644 index 0000000000..9c14cccc93 --- /dev/null +++ b/codex-rs/tui/src/scroll_event_helper.rs @@ -0,0 +1,71 @@ +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; +use std::sync::mpsc::Sender; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use crate::app_event::AppEvent; + +pub(crate) struct ScrollEventHelper { + app_event_tx: Sender, + scroll_delta: Arc, + timer_scheduled: Arc, +} + +/// How long to wait after the first scroll event before sending the +/// accumulated scroll delta to the main thread. +const DEBOUNCE_WINDOW: Duration = Duration::from_millis(100); + +/// Utility to debounce scroll events so we can determine estimate the +/// "magnitude" of the scroll event by accumulating them over a short window. +impl ScrollEventHelper { + pub(crate) fn new(app_event_tx: Sender) -> Self { + Self { + app_event_tx, + scroll_delta: Arc::new(AtomicI32::new(0)), + timer_scheduled: Arc::new(AtomicBool::new(false)), + } + } + + pub(crate) fn scroll_up(&self) { + self.scroll_delta.fetch_sub(1, Ordering::Relaxed); + self.schedule_notification(); + } + + pub(crate) fn scroll_down(&self) { + self.scroll_delta.fetch_add(1, Ordering::Relaxed); + self.schedule_notification(); + } + + /// Starts a one-shot timer **only once** per burst of wheel events. + fn schedule_notification(&self) { + // If the timer is already scheduled, do nothing. + if self + .timer_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + + // Otherwise, schedule a new timer. + let tx = self.app_event_tx.clone(); + let delta = Arc::clone(&self.scroll_delta); + let timer_flag = Arc::clone(&self.timer_scheduled); + + thread::spawn(move || { + thread::sleep(DEBOUNCE_WINDOW); + + let accumulated = delta.swap(0, Ordering::SeqCst); + // Only emit if something really happened. + if accumulated != 0 { + let _ = tx.send(AppEvent::Scroll(accumulated)); + } + + // Allow a new timer to be started on the next wheel event. + timer_flag.store(false, Ordering::SeqCst); + }); + } +} diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 0753dcb07a..8cc54460a4 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -2,6 +2,8 @@ use std::io::stdout; use std::io::Stdout; use std::io::{self}; +use crossterm::event::DisableMouseCapture; +use crossterm::event::EnableMouseCapture; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -16,6 +18,7 @@ pub type Tui = Terminal>; /// Initialize the terminal pub fn init() -> io::Result { execute!(stdout(), EnterAlternateScreen)?; + execute!(stdout(), EnableMouseCapture)?; enable_raw_mode()?; set_panic_hook(); Terminal::new(CrosstermBackend::new(stdout())) @@ -31,6 +34,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> io::Result<()> { + execute!(stdout(), DisableMouseCapture)?; execute!(stdout(), LeaveAlternateScreen)?; disable_raw_mode()?; Ok(()) From b90cf2a778c6403a0a457d81debed9b67b8f8f97 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 20:10:52 -0700 Subject: [PATCH 63/84] feat(tui-rs): add support for mousewheel scrolling --- codex-rs/tui/src/app.rs | 34 +++++++-- codex-rs/tui/src/app_event.rs | 7 ++ codex-rs/tui/src/chatwidget.rs | 17 +++++ .../tui/src/conversation_history_widget.rs | 31 +++++--- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/scroll_event_helper.rs | 71 +++++++++++++++++++ codex-rs/tui/src/tui.rs | 4 ++ 7 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 codex-rs/tui/src/scroll_event_helper.rs diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..26a7074b5b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2,6 +2,7 @@ use crate::app_event::AppEvent; use crate::chatwidget::ChatWidget; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; +use crate::scroll_event_helper::ScrollEventHelper; use crate::tui; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; @@ -10,6 +11,8 @@ use codex_core::protocol::SandboxPolicy; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; use std::sync::mpsc::Sender; @@ -39,6 +42,7 @@ impl App<'_> { model: Option, ) -> Self { let (app_event_tx, app_event_rx) = channel(); + let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -49,10 +53,21 @@ impl App<'_> { let app_event = match event { crossterm::event::Event::Key(key_event) => AppEvent::KeyEvent(key_event), crossterm::event::Event::Resize(_, _) => AppEvent::Redraw, - crossterm::event::Event::FocusGained - | crossterm::event::Event::FocusLost - | crossterm::event::Event::Mouse(_) - | crossterm::event::Event::Paste(_) => { + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollUp, + .. + }) => { + scroll_event_helper.scroll_up(); + continue; + } + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollDown, + .. + }) => { + scroll_event_helper.scroll_down(); + continue; + } + _ => { continue; } }; @@ -125,6 +140,9 @@ impl App<'_> { } }; } + AppEvent::Scroll(scroll_delta) => { + self.dispatch_scroll_event(scroll_delta); + } AppEvent::CodexEvent(event) => { self.dispatch_codex_event(event); } @@ -184,6 +202,14 @@ impl App<'_> { } } + fn dispatch_scroll_event(&mut self, scroll_delta: i32) { + if matches!(self.app_state, AppState::Chat) { + if let Err(e) = self.chat_widget.handle_scroll_delta(scroll_delta) { + tracing::error!("SendError: {e}"); + } + } + } + fn dispatch_codex_event(&mut self, event: Event) { if matches!(self.app_state, AppState::Chat) { if let Err(e) = self.chat_widget.handle_codex_event(event) { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index bb8efb8e15..2b320375be 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -3,8 +3,15 @@ use crossterm::event::KeyEvent; pub(crate) enum AppEvent { CodexEvent(Event), + Redraw, + KeyEvent(KeyEvent), + + /// Scroll event with a value representing the "scroll delta" as the net + /// scroll up/down events within a short time window. + Scroll(i32), + /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..64cb896d93 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -364,6 +364,23 @@ impl ChatWidget<'_> { Ok(()) } + pub(crate) fn handle_scroll_delta( + &mut self, + scroll_delta: i32, + ) -> std::result::Result<(), std::sync::mpsc::SendError> { + // If the user is trying to scroll exactly one line, we let them, but + // otherwise we assume they are trying to scroll in larger increments. + let magnified_scroll_delta = if scroll_delta == 1 { + 1 + } else { + // Play with this: perhaps it should be non-linear? + scroll_delta * 2 + }; + self.conversation_history.scroll(magnified_scroll_delta); + self.request_redraw()?; + Ok(()) + } + /// Forward an `Op` directly to codex. pub(crate) fn submit_op(&self, op: Op) { if let Err(e) = self.codex_op_tx.send(op) { diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index c8f6906169..27b5e9b3cf 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -40,11 +40,11 @@ impl ConversationHistoryWidget { pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) -> bool { match key_event.code { KeyCode::Up | KeyCode::Char('k') => { - self.scroll_up(); + self.scroll_up(1); true } KeyCode::Down | KeyCode::Char('j') => { - self.scroll_down(); + self.scroll_down(1); true } KeyCode::PageUp | KeyCode::Char('b') | KeyCode::Char('u') | KeyCode::Char('U') => { @@ -59,9 +59,18 @@ impl ConversationHistoryWidget { } } - fn scroll_up(&mut self) { - // If a user is scrolling up from the "stick to bottom" mode, we - // need to scroll them back such that they move just one line up. + /// Negative delta scrolls up; positive delta scrolls down. + pub(crate) fn scroll(&mut self, delta: i32) { + match delta.cmp(&0) { + std::cmp::Ordering::Less => self.scroll_up(-delta as u32), + std::cmp::Ordering::Greater => self.scroll_down(delta as u32), + std::cmp::Ordering::Equal => {} + } + } + + fn scroll_up(&mut self, num_lines: u32) { + // If a user is scrolling up from the "stick to bottom" mode, we need to + // map this to a specific scroll position so we can caluate the delta. // This requires us to care about how tall the screen is. if self.scroll_position == usize::MAX { self.scroll_position = self @@ -70,24 +79,26 @@ impl ConversationHistoryWidget { .saturating_sub(self.last_viewport_height.get()); } - self.scroll_position = self.scroll_position.saturating_sub(1); + self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); } - fn scroll_down(&mut self) { + fn scroll_down(&mut self, num_lines: u32) { // If we're already pinned to the bottom there's nothing to do. if self.scroll_position == usize::MAX { return; } let viewport_height = self.last_viewport_height.get().max(1); - let num_lines = self.num_rendered_lines.get(); + let num_rendered_lines = self.num_rendered_lines.get(); // Compute the maximum explicit scroll offset that still shows a full // viewport. This mirrors the calculation in `scroll_page_down()` and // in the render path. - let max_scroll = num_lines.saturating_sub(viewport_height).saturating_add(1); + let max_scroll = num_rendered_lines + .saturating_sub(viewport_height) + .saturating_add(1); - let new_pos = self.scroll_position.saturating_add(1); + let new_pos = self.scroll_position.saturating_add(num_lines as usize); if new_pos >= max_scroll { // Reached (or passed) the bottom – switch to stick‑to‑bottom mode diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..7361663bb4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -21,6 +21,7 @@ mod exec_command; mod git_warning_screen; mod history_cell; mod log_layer; +mod scroll_event_helper; mod status_indicator_widget; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/scroll_event_helper.rs b/codex-rs/tui/src/scroll_event_helper.rs new file mode 100644 index 0000000000..9c14cccc93 --- /dev/null +++ b/codex-rs/tui/src/scroll_event_helper.rs @@ -0,0 +1,71 @@ +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; +use std::sync::mpsc::Sender; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use crate::app_event::AppEvent; + +pub(crate) struct ScrollEventHelper { + app_event_tx: Sender, + scroll_delta: Arc, + timer_scheduled: Arc, +} + +/// How long to wait after the first scroll event before sending the +/// accumulated scroll delta to the main thread. +const DEBOUNCE_WINDOW: Duration = Duration::from_millis(100); + +/// Utility to debounce scroll events so we can determine estimate the +/// "magnitude" of the scroll event by accumulating them over a short window. +impl ScrollEventHelper { + pub(crate) fn new(app_event_tx: Sender) -> Self { + Self { + app_event_tx, + scroll_delta: Arc::new(AtomicI32::new(0)), + timer_scheduled: Arc::new(AtomicBool::new(false)), + } + } + + pub(crate) fn scroll_up(&self) { + self.scroll_delta.fetch_sub(1, Ordering::Relaxed); + self.schedule_notification(); + } + + pub(crate) fn scroll_down(&self) { + self.scroll_delta.fetch_add(1, Ordering::Relaxed); + self.schedule_notification(); + } + + /// Starts a one-shot timer **only once** per burst of wheel events. + fn schedule_notification(&self) { + // If the timer is already scheduled, do nothing. + if self + .timer_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + + // Otherwise, schedule a new timer. + let tx = self.app_event_tx.clone(); + let delta = Arc::clone(&self.scroll_delta); + let timer_flag = Arc::clone(&self.timer_scheduled); + + thread::spawn(move || { + thread::sleep(DEBOUNCE_WINDOW); + + let accumulated = delta.swap(0, Ordering::SeqCst); + // Only emit if something really happened. + if accumulated != 0 { + let _ = tx.send(AppEvent::Scroll(accumulated)); + } + + // Allow a new timer to be started on the next wheel event. + timer_flag.store(false, Ordering::SeqCst); + }); + } +} diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 0753dcb07a..8cc54460a4 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -2,6 +2,8 @@ use std::io::stdout; use std::io::Stdout; use std::io::{self}; +use crossterm::event::DisableMouseCapture; +use crossterm::event::EnableMouseCapture; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -16,6 +18,7 @@ pub type Tui = Terminal>; /// Initialize the terminal pub fn init() -> io::Result { execute!(stdout(), EnterAlternateScreen)?; + execute!(stdout(), EnableMouseCapture)?; enable_raw_mode()?; set_panic_hook(); Terminal::new(CrosstermBackend::new(stdout())) @@ -31,6 +34,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> io::Result<()> { + execute!(stdout(), DisableMouseCapture)?; execute!(stdout(), LeaveAlternateScreen)?; disable_raw_mode()?; Ok(()) From 461ab550ce0d88af2b3bd638169345381b7ea84a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 20:10:52 -0700 Subject: [PATCH 64/84] feat(tui-rs): add support for mousewheel scrolling --- codex-rs/tui/src/app.rs | 34 +++++++-- codex-rs/tui/src/app_event.rs | 7 ++ codex-rs/tui/src/chatwidget.rs | 17 +++++ .../tui/src/conversation_history_widget.rs | 31 ++++++--- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/scroll_event_helper.rs | 69 +++++++++++++++++++ codex-rs/tui/src/tui.rs | 4 ++ 7 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 codex-rs/tui/src/scroll_event_helper.rs diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..26a7074b5b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2,6 +2,7 @@ use crate::app_event::AppEvent; use crate::chatwidget::ChatWidget; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; +use crate::scroll_event_helper::ScrollEventHelper; use crate::tui; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; @@ -10,6 +11,8 @@ use codex_core::protocol::SandboxPolicy; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; use std::sync::mpsc::Sender; @@ -39,6 +42,7 @@ impl App<'_> { model: Option, ) -> Self { let (app_event_tx, app_event_rx) = channel(); + let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -49,10 +53,21 @@ impl App<'_> { let app_event = match event { crossterm::event::Event::Key(key_event) => AppEvent::KeyEvent(key_event), crossterm::event::Event::Resize(_, _) => AppEvent::Redraw, - crossterm::event::Event::FocusGained - | crossterm::event::Event::FocusLost - | crossterm::event::Event::Mouse(_) - | crossterm::event::Event::Paste(_) => { + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollUp, + .. + }) => { + scroll_event_helper.scroll_up(); + continue; + } + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollDown, + .. + }) => { + scroll_event_helper.scroll_down(); + continue; + } + _ => { continue; } }; @@ -125,6 +140,9 @@ impl App<'_> { } }; } + AppEvent::Scroll(scroll_delta) => { + self.dispatch_scroll_event(scroll_delta); + } AppEvent::CodexEvent(event) => { self.dispatch_codex_event(event); } @@ -184,6 +202,14 @@ impl App<'_> { } } + fn dispatch_scroll_event(&mut self, scroll_delta: i32) { + if matches!(self.app_state, AppState::Chat) { + if let Err(e) = self.chat_widget.handle_scroll_delta(scroll_delta) { + tracing::error!("SendError: {e}"); + } + } + } + fn dispatch_codex_event(&mut self, event: Event) { if matches!(self.app_state, AppState::Chat) { if let Err(e) = self.chat_widget.handle_codex_event(event) { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index bb8efb8e15..2b320375be 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -3,8 +3,15 @@ use crossterm::event::KeyEvent; pub(crate) enum AppEvent { CodexEvent(Event), + Redraw, + KeyEvent(KeyEvent), + + /// Scroll event with a value representing the "scroll delta" as the net + /// scroll up/down events within a short time window. + Scroll(i32), + /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..64cb896d93 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -364,6 +364,23 @@ impl ChatWidget<'_> { Ok(()) } + pub(crate) fn handle_scroll_delta( + &mut self, + scroll_delta: i32, + ) -> std::result::Result<(), std::sync::mpsc::SendError> { + // If the user is trying to scroll exactly one line, we let them, but + // otherwise we assume they are trying to scroll in larger increments. + let magnified_scroll_delta = if scroll_delta == 1 { + 1 + } else { + // Play with this: perhaps it should be non-linear? + scroll_delta * 2 + }; + self.conversation_history.scroll(magnified_scroll_delta); + self.request_redraw()?; + Ok(()) + } + /// Forward an `Op` directly to codex. pub(crate) fn submit_op(&self, op: Op) { if let Err(e) = self.codex_op_tx.send(op) { diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index c8f6906169..27b5e9b3cf 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -40,11 +40,11 @@ impl ConversationHistoryWidget { pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) -> bool { match key_event.code { KeyCode::Up | KeyCode::Char('k') => { - self.scroll_up(); + self.scroll_up(1); true } KeyCode::Down | KeyCode::Char('j') => { - self.scroll_down(); + self.scroll_down(1); true } KeyCode::PageUp | KeyCode::Char('b') | KeyCode::Char('u') | KeyCode::Char('U') => { @@ -59,9 +59,18 @@ impl ConversationHistoryWidget { } } - fn scroll_up(&mut self) { - // If a user is scrolling up from the "stick to bottom" mode, we - // need to scroll them back such that they move just one line up. + /// Negative delta scrolls up; positive delta scrolls down. + pub(crate) fn scroll(&mut self, delta: i32) { + match delta.cmp(&0) { + std::cmp::Ordering::Less => self.scroll_up(-delta as u32), + std::cmp::Ordering::Greater => self.scroll_down(delta as u32), + std::cmp::Ordering::Equal => {} + } + } + + fn scroll_up(&mut self, num_lines: u32) { + // If a user is scrolling up from the "stick to bottom" mode, we need to + // map this to a specific scroll position so we can caluate the delta. // This requires us to care about how tall the screen is. if self.scroll_position == usize::MAX { self.scroll_position = self @@ -70,24 +79,26 @@ impl ConversationHistoryWidget { .saturating_sub(self.last_viewport_height.get()); } - self.scroll_position = self.scroll_position.saturating_sub(1); + self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); } - fn scroll_down(&mut self) { + fn scroll_down(&mut self, num_lines: u32) { // If we're already pinned to the bottom there's nothing to do. if self.scroll_position == usize::MAX { return; } let viewport_height = self.last_viewport_height.get().max(1); - let num_lines = self.num_rendered_lines.get(); + let num_rendered_lines = self.num_rendered_lines.get(); // Compute the maximum explicit scroll offset that still shows a full // viewport. This mirrors the calculation in `scroll_page_down()` and // in the render path. - let max_scroll = num_lines.saturating_sub(viewport_height).saturating_add(1); + let max_scroll = num_rendered_lines + .saturating_sub(viewport_height) + .saturating_add(1); - let new_pos = self.scroll_position.saturating_add(1); + let new_pos = self.scroll_position.saturating_add(num_lines as usize); if new_pos >= max_scroll { // Reached (or passed) the bottom – switch to stick‑to‑bottom mode diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..7361663bb4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -21,6 +21,7 @@ mod exec_command; mod git_warning_screen; mod history_cell; mod log_layer; +mod scroll_event_helper; mod status_indicator_widget; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/scroll_event_helper.rs b/codex-rs/tui/src/scroll_event_helper.rs new file mode 100644 index 0000000000..b68d487c9c --- /dev/null +++ b/codex-rs/tui/src/scroll_event_helper.rs @@ -0,0 +1,69 @@ +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; +use std::sync::mpsc::Sender; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use crate::app_event::AppEvent; + +pub(crate) struct ScrollEventHelper { + app_event_tx: Sender, + scroll_delta: Arc, + timer_scheduled: Arc, +} + +/// How long to wait after the first scroll event before sending the +/// accumulated scroll delta to the main thread. +const DEBOUNCE_WINDOW: Duration = Duration::from_millis(100); + +/// Utility to debounce scroll events so we can determine estimate the +/// "magnitude" of the scroll event by accumulating them over a short window. +impl ScrollEventHelper { + pub(crate) fn new(app_event_tx: Sender) -> Self { + Self { + app_event_tx, + scroll_delta: Arc::new(AtomicI32::new(0)), + timer_scheduled: Arc::new(AtomicBool::new(false)), + } + } + + pub(crate) fn scroll_up(&self) { + self.scroll_delta.fetch_sub(1, Ordering::Relaxed); + self.schedule_notification(); + } + + pub(crate) fn scroll_down(&self) { + self.scroll_delta.fetch_add(1, Ordering::Relaxed); + self.schedule_notification(); + } + + /// Starts a one-shot timer **only once** per burst of wheel events. + fn schedule_notification(&self) { + // If the timer is already scheduled, do nothing. + if self + .timer_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + + // Otherwise, schedule a new timer. + let tx = self.app_event_tx.clone(); + let delta = Arc::clone(&self.scroll_delta); + let timer_flag = Arc::clone(&self.timer_scheduled); + + thread::spawn(move || { + thread::sleep(DEBOUNCE_WINDOW); + + let accumulated = delta.swap(0, Ordering::SeqCst); + if accumulated != 0 { + let _ = tx.send(AppEvent::Scroll(accumulated)); + } + + timer_flag.store(false, Ordering::SeqCst); + }); + } +} diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 0753dcb07a..8cc54460a4 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -2,6 +2,8 @@ use std::io::stdout; use std::io::Stdout; use std::io::{self}; +use crossterm::event::DisableMouseCapture; +use crossterm::event::EnableMouseCapture; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -16,6 +18,7 @@ pub type Tui = Terminal>; /// Initialize the terminal pub fn init() -> io::Result { execute!(stdout(), EnterAlternateScreen)?; + execute!(stdout(), EnableMouseCapture)?; enable_raw_mode()?; set_panic_hook(); Terminal::new(CrosstermBackend::new(stdout())) @@ -31,6 +34,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> io::Result<()> { + execute!(stdout(), DisableMouseCapture)?; execute!(stdout(), LeaveAlternateScreen)?; disable_raw_mode()?; Ok(()) From 73eb766637621dae7ae62fd09cfbf4369367d643 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 20:10:52 -0700 Subject: [PATCH 65/84] feat(tui-rs): add support for mousewheel scrolling --- codex-rs/tui/src/app.rs | 34 +++++++- codex-rs/tui/src/app_event.rs | 7 ++ codex-rs/tui/src/chatwidget.rs | 17 ++++ .../tui/src/conversation_history_widget.rs | 31 +++++--- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/scroll_event_helper.rs | 77 +++++++++++++++++++ codex-rs/tui/src/tui.rs | 4 + 7 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 codex-rs/tui/src/scroll_event_helper.rs diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..26a7074b5b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2,6 +2,7 @@ use crate::app_event::AppEvent; use crate::chatwidget::ChatWidget; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; +use crate::scroll_event_helper::ScrollEventHelper; use crate::tui; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; @@ -10,6 +11,8 @@ use codex_core::protocol::SandboxPolicy; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; use std::sync::mpsc::Sender; @@ -39,6 +42,7 @@ impl App<'_> { model: Option, ) -> Self { let (app_event_tx, app_event_rx) = channel(); + let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -49,10 +53,21 @@ impl App<'_> { let app_event = match event { crossterm::event::Event::Key(key_event) => AppEvent::KeyEvent(key_event), crossterm::event::Event::Resize(_, _) => AppEvent::Redraw, - crossterm::event::Event::FocusGained - | crossterm::event::Event::FocusLost - | crossterm::event::Event::Mouse(_) - | crossterm::event::Event::Paste(_) => { + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollUp, + .. + }) => { + scroll_event_helper.scroll_up(); + continue; + } + crossterm::event::Event::Mouse(MouseEvent { + kind: MouseEventKind::ScrollDown, + .. + }) => { + scroll_event_helper.scroll_down(); + continue; + } + _ => { continue; } }; @@ -125,6 +140,9 @@ impl App<'_> { } }; } + AppEvent::Scroll(scroll_delta) => { + self.dispatch_scroll_event(scroll_delta); + } AppEvent::CodexEvent(event) => { self.dispatch_codex_event(event); } @@ -184,6 +202,14 @@ impl App<'_> { } } + fn dispatch_scroll_event(&mut self, scroll_delta: i32) { + if matches!(self.app_state, AppState::Chat) { + if let Err(e) = self.chat_widget.handle_scroll_delta(scroll_delta) { + tracing::error!("SendError: {e}"); + } + } + } + fn dispatch_codex_event(&mut self, event: Event) { if matches!(self.app_state, AppState::Chat) { if let Err(e) = self.chat_widget.handle_codex_event(event) { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index bb8efb8e15..2b320375be 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -3,8 +3,15 @@ use crossterm::event::KeyEvent; pub(crate) enum AppEvent { CodexEvent(Event), + Redraw, + KeyEvent(KeyEvent), + + /// Scroll event with a value representing the "scroll delta" as the net + /// scroll up/down events within a short time window. + Scroll(i32), + /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..64cb896d93 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -364,6 +364,23 @@ impl ChatWidget<'_> { Ok(()) } + pub(crate) fn handle_scroll_delta( + &mut self, + scroll_delta: i32, + ) -> std::result::Result<(), std::sync::mpsc::SendError> { + // If the user is trying to scroll exactly one line, we let them, but + // otherwise we assume they are trying to scroll in larger increments. + let magnified_scroll_delta = if scroll_delta == 1 { + 1 + } else { + // Play with this: perhaps it should be non-linear? + scroll_delta * 2 + }; + self.conversation_history.scroll(magnified_scroll_delta); + self.request_redraw()?; + Ok(()) + } + /// Forward an `Op` directly to codex. pub(crate) fn submit_op(&self, op: Op) { if let Err(e) = self.codex_op_tx.send(op) { diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index c8f6906169..27b5e9b3cf 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -40,11 +40,11 @@ impl ConversationHistoryWidget { pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) -> bool { match key_event.code { KeyCode::Up | KeyCode::Char('k') => { - self.scroll_up(); + self.scroll_up(1); true } KeyCode::Down | KeyCode::Char('j') => { - self.scroll_down(); + self.scroll_down(1); true } KeyCode::PageUp | KeyCode::Char('b') | KeyCode::Char('u') | KeyCode::Char('U') => { @@ -59,9 +59,18 @@ impl ConversationHistoryWidget { } } - fn scroll_up(&mut self) { - // If a user is scrolling up from the "stick to bottom" mode, we - // need to scroll them back such that they move just one line up. + /// Negative delta scrolls up; positive delta scrolls down. + pub(crate) fn scroll(&mut self, delta: i32) { + match delta.cmp(&0) { + std::cmp::Ordering::Less => self.scroll_up(-delta as u32), + std::cmp::Ordering::Greater => self.scroll_down(delta as u32), + std::cmp::Ordering::Equal => {} + } + } + + fn scroll_up(&mut self, num_lines: u32) { + // If a user is scrolling up from the "stick to bottom" mode, we need to + // map this to a specific scroll position so we can caluate the delta. // This requires us to care about how tall the screen is. if self.scroll_position == usize::MAX { self.scroll_position = self @@ -70,24 +79,26 @@ impl ConversationHistoryWidget { .saturating_sub(self.last_viewport_height.get()); } - self.scroll_position = self.scroll_position.saturating_sub(1); + self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); } - fn scroll_down(&mut self) { + fn scroll_down(&mut self, num_lines: u32) { // If we're already pinned to the bottom there's nothing to do. if self.scroll_position == usize::MAX { return; } let viewport_height = self.last_viewport_height.get().max(1); - let num_lines = self.num_rendered_lines.get(); + let num_rendered_lines = self.num_rendered_lines.get(); // Compute the maximum explicit scroll offset that still shows a full // viewport. This mirrors the calculation in `scroll_page_down()` and // in the render path. - let max_scroll = num_lines.saturating_sub(viewport_height).saturating_add(1); + let max_scroll = num_rendered_lines + .saturating_sub(viewport_height) + .saturating_add(1); - let new_pos = self.scroll_position.saturating_add(1); + let new_pos = self.scroll_position.saturating_add(num_lines as usize); if new_pos >= max_scroll { // Reached (or passed) the bottom – switch to stick‑to‑bottom mode diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..7361663bb4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -21,6 +21,7 @@ mod exec_command; mod git_warning_screen; mod history_cell; mod log_layer; +mod scroll_event_helper; mod status_indicator_widget; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/scroll_event_helper.rs b/codex-rs/tui/src/scroll_event_helper.rs new file mode 100644 index 0000000000..7c358157df --- /dev/null +++ b/codex-rs/tui/src/scroll_event_helper.rs @@ -0,0 +1,77 @@ +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; +use std::sync::mpsc::Sender; +use std::sync::Arc; + +use tokio::runtime::Handle; +use tokio::time::sleep; +use tokio::time::Duration; + +use crate::app_event::AppEvent; + +pub(crate) struct ScrollEventHelper { + app_event_tx: Sender, + scroll_delta: Arc, + timer_scheduled: Arc, + runtime: Handle, +} + +/// How long to wait after the first scroll event before sending the +/// accumulated scroll delta to the main thread. +const DEBOUNCE_WINDOW: Duration = Duration::from_millis(100); + +/// Utility to debounce scroll events so we can determine the **magnitude** of +/// each scroll burst by accumulating individual wheel events over a short +/// window. The debounce timer now runs on Tokio so we avoid spinning up a new +/// operating-system thread for every burst. +impl ScrollEventHelper { + pub(crate) fn new(app_event_tx: Sender) -> Self { + Self { + app_event_tx, + scroll_delta: Arc::new(AtomicI32::new(0)), + timer_scheduled: Arc::new(AtomicBool::new(false)), + runtime: Handle::current(), + } + } + + pub(crate) fn scroll_up(&self) { + self.scroll_delta.fetch_sub(1, Ordering::Relaxed); + self.schedule_notification(); + } + + pub(crate) fn scroll_down(&self) { + self.scroll_delta.fetch_add(1, Ordering::Relaxed); + self.schedule_notification(); + } + + /// Starts a one-shot timer **only once** per burst of wheel events. + fn schedule_notification(&self) { + // If the timer is already scheduled, do nothing. + if self + .timer_scheduled + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + + // Otherwise, schedule a new timer. + let tx = self.app_event_tx.clone(); + let delta = Arc::clone(&self.scroll_delta); + let timer_flag = Arc::clone(&self.timer_scheduled); + + // Use self.runtime instead of tokio::spawn() because the calling thread + // in app.rs is not part of the Tokio runtime: it is a plain OS thread. + self.runtime.spawn(async move { + sleep(DEBOUNCE_WINDOW).await; + + let accumulated = delta.swap(0, Ordering::SeqCst); + if accumulated != 0 { + let _ = tx.send(AppEvent::Scroll(accumulated)); + } + + timer_flag.store(false, Ordering::SeqCst); + }); + } +} diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 0753dcb07a..8cc54460a4 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -2,6 +2,8 @@ use std::io::stdout; use std::io::Stdout; use std::io::{self}; +use crossterm::event::DisableMouseCapture; +use crossterm::event::EnableMouseCapture; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -16,6 +18,7 @@ pub type Tui = Terminal>; /// Initialize the terminal pub fn init() -> io::Result { execute!(stdout(), EnterAlternateScreen)?; + execute!(stdout(), EnableMouseCapture)?; enable_raw_mode()?; set_panic_hook(); Terminal::new(CrosstermBackend::new(stdout())) @@ -31,6 +34,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> io::Result<()> { + execute!(stdout(), DisableMouseCapture)?; execute!(stdout(), LeaveAlternateScreen)?; disable_raw_mode()?; Ok(()) From 9717ac87c1b8d8d6ba663264d3e62815e957c651 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 66/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 12 ++++++- codex-rs/core/src/codex.rs | 38 +++++++++++++++++++-- codex-rs/core/src/codex_wrapper.rs | 2 ++ codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/protocol.rs | 3 ++ codex-rs/core/src/zdr_transcript.rs | 26 ++++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 6 +++- codex-rs/exec/src/lib.rs | 3 +- codex-rs/repl/src/lib.rs | 2 ++ codex-rs/tui/src/chatwidget.rs | 3 +- 13 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..c4460be9aa 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -33,11 +33,17 @@ use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn #[derive(Default, Debug, Clone)] pub struct Prompt { + /// Conversation context input items pub input: Vec, + /// Optional previous response ID (when storage is enabled) pub prev_id: Option, + /// Optional initial instructions (only sent on first turn) pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store) + pub store: bool, } #[derive(Debug)] @@ -58,7 +64,9 @@ struct Payload<'a> { reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, - stream: bool, + store: bool, + /// Stream responses via SSE + pub stream: bool, } #[derive(Debug, Serialize)] @@ -152,6 +160,8 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + // store response on server side when enabled + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..08b6df733c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -190,6 +191,9 @@ impl Recorder { } } +/// Context for an initialized model agent +/// +/// A session has at most 1 running task at a time, and can be interrupted by user input. /// Context for an initialized model agent /// /// A session has at most 1 running task at a time, and can be interrupted by user input. @@ -201,6 +205,9 @@ struct Session { instructions: Option, approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + /// When true, omit `previous_response_id` in requests and send full context + disable_response_storage: bool, + /// Additional writable roots for sandbox execution writable_roots: Mutex>, state: Mutex, @@ -214,6 +221,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +407,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +498,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +510,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -511,6 +528,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, writable_roots: Mutex::new(get_writable_roots()), state: Mutex::new(state), })); @@ -592,12 +610,21 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { let pending_input = sess.get_pending_input(); turn_input.splice(0..0, pending_input); + // If sess.state.transcript.is_some(), it should be written back into + // the prompt. match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { if turn_output.is_empty() { debug!("Turn completed"); break; } + + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + for item in &turn_output { + transcript.add_item(item.clone()); + } + } + turn_input = turn_output; } Err(e) => { @@ -626,19 +653,26 @@ async fn run_turn( sub_id: String, input: Vec, ) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it let prev_id = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + if sess.disable_response_storage { + None + } else { + state.previous_response_id.clone() + } }; let instructions = match prev_id { Some(_) => None, None => sess.instructions.clone(), }; + // Build prompt payload, including store flag let prompt = Prompt { input, prev_id, instructions, + store: !sess.disable_response_storage, }; let mut retries = 0; diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..43c2d2620f --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,26 @@ +use crate::models::ResponseInputItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + pub(crate) fn add_item(&mut self, item: ResponseInputItem) { + if is_api_message(&item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item); + } + } +} + +fn is_api_message(message: &ResponseInputItem) -> bool { + !matches!(message, ResponseInputItem::Message { role, .. } if role.as_str() == "system") +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..938de29d23 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, ArgAction}; use std::path::PathBuf; #[derive(Parser, Debug)] @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server-side response storage (omits previous_response_id and controls store flag) + #[arg(long = "disable-response-storage", action = ArgAction::SetTrue, default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..e9a8518d7c 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -32,6 +32,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let Cli { skip_git_repo_check, + disable_response_storage, model, images, prompt, @@ -51,7 +52,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + codex_wrapper::init_codex(approval_policy, sandbox_policy, disable_response_storage, model).await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..7f3cd4a414 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + // by default, use server-side storage + disable_response_storage: false, }, }; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..d8228a131a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -63,8 +63,9 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { + // Initialize session; storage enabled by default let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { + match init_codex(approval_policy, sandbox_policy, false, model).await { Ok(vals) => vals, Err(e) => { // TODO(mbolin): This error needs to be surfaced to the user. From 79caa9221737ee0cd7ab0b7abe575beef0c51a16 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 67/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 17 +++-- codex-rs/core/src/codex.rs | 80 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 44 ++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 6 +- codex-rs/exec/src/lib.rs | 3 +- codex-rs/repl/src/lib.rs | 2 + codex-rs/tui/src/chatwidget.rs | 3 +- 14 files changed, 156 insertions(+), 19 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..bd2453348e 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items + pub input: Vec, + /// Optional previous response ID (when storage is enabled) pub prev_id: Option, + /// Optional initial instructions (only sent on first turn) pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store) + pub store: bool, } #[derive(Debug)] @@ -51,14 +56,16 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, - stream: bool, + store: bool, + /// Stream responses via SSE + pub stream: bool, } #[derive(Debug, Serialize)] @@ -152,6 +159,8 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + // store response on server side when enabled + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..017e8052d2 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -190,6 +191,9 @@ impl Recorder { } } +/// Context for an initialized model agent +/// +/// A session has at most 1 running task at a time, and can be interrupted by user input. /// Context for an initialized model agent /// /// A session has at most 1 running task at a time, and can be interrupted by user input. @@ -201,6 +205,9 @@ struct Session { instructions: Option, approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + /// When true, omit `previous_response_id` in requests and send full context + disable_response_storage: bool, + /// Additional writable roots for sandbox execution writable_roots: Mutex>, state: Mutex, @@ -214,6 +221,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +407,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +498,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +510,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -511,6 +528,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, writable_roots: Mutex::new(get_writable_roots()), state: Mutex::new(state), })); @@ -587,10 +605,23 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut turn_input: Vec = + if let Some(transcript) = &sess.state.lock().unwrap().zdr_transcript { + // If we are using ZDR, we need to send the transcript with every turn. + transcript.contents() + } else { + Vec::new() + }; + + turn_input.extend(pending_response_input.drain(..).map(ResponseItem::from)); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + turn_input.extend(pending_input); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { @@ -598,7 +629,17 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { debug!("Turn completed"); break; } - turn_input = turn_output; + + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + let num_added = transcript.record_items(turn_output.iter().map(|i| &i.item)); + if num_added == 0 { + debug!("Turn completed"); + break; + } + } + + pending_response_input = + turn_output.into_iter().filter_map(|i| i.response).collect(); } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +665,28 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it let prev_id = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + if sess.disable_response_storage { + None + } else { + state.previous_response_id.clone() + } }; let instructions = match prev_id { Some(_) => None, None => sess.instructions.clone(), }; + // Build prompt payload, including store flag let prompt = Prompt { input, prev_id, instructions, + store: !sess.disable_response_storage, }; let mut retries = 0; @@ -676,11 +724,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +751,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..f97133ce69 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,44 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items<'a, I>(&mut self, items: I) -> usize + where + I: IntoIterator, + { + let mut count = 0; + for item in items { + if is_api_message(item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + count += 1; + } + } + count + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + !matches!(message, ResponseItem::Message { role, .. } if role.as_str() == "system") +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..938de29d23 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, ArgAction}; use std::path::PathBuf; #[derive(Parser, Debug)] @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server-side response storage (omits previous_response_id and controls store flag) + #[arg(long = "disable-response-storage", action = ArgAction::SetTrue, default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..e9a8518d7c 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -32,6 +32,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let Cli { skip_git_repo_check, + disable_response_storage, model, images, prompt, @@ -51,7 +52,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + codex_wrapper::init_codex(approval_policy, sandbox_policy, disable_response_storage, model).await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..7f3cd4a414 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + // by default, use server-side storage + disable_response_storage: false, }, }; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..d8228a131a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -63,8 +63,9 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { + // Initialize session; storage enabled by default let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { + match init_codex(approval_policy, sandbox_policy, false, model).await { Ok(vals) => vals, Err(e) => { // TODO(mbolin): This error needs to be surfaced to the user. From 6538d9c5922d9dc18c8234f08c9ec0bc352e3cda Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 68/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 14 +++- codex-rs/core/src/codex.rs | 74 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 44 ++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 6 +- codex-rs/exec/src/lib.rs | 3 +- codex-rs/repl/src/lib.rs | 2 + codex-rs/tui/src/chatwidget.rs | 3 +- 14 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..b61741f303 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -51,13 +56,15 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + store: bool, stream: bool, } @@ -152,6 +159,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..5af1080d2f 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,10 +598,23 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut turn_input: Vec = + if let Some(transcript) = &sess.state.lock().unwrap().zdr_transcript { + // If we are using ZDR, we need to send the transcript with every turn. + transcript.contents() + } else { + Vec::new() + }; + + turn_input.extend(pending_response_input.drain(..).map(ResponseItem::from)); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + turn_input.extend(pending_input); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { @@ -598,7 +622,17 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { debug!("Turn completed"); break; } - turn_input = turn_output; + + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + let num_added = transcript.record_items(turn_output.iter().map(|i| &i.item)); + if num_added == 0 { + debug!("Turn completed"); + break; + } + } + + pending_response_input = + turn_output.into_iter().filter_map(|i| i.response).collect(); } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +658,27 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + ( + state.previous_response_id.clone(), + state.zdr_transcript.is_none(), + ) }; let instructions = match prev_id { Some(_) => None, None => sess.instructions.clone(), }; + // Build prompt payload, including store flag let prompt = Prompt { input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +716,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +743,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..f97133ce69 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,44 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items<'a, I>(&mut self, items: I) -> usize + where + I: IntoIterator, + { + let mut count = 0; + for item in items { + if is_api_message(item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + count += 1; + } + } + count + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + !matches!(message, ResponseItem::Message { role, .. } if role.as_str() == "system") +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..938de29d23 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, ArgAction}; use std::path::PathBuf; #[derive(Parser, Debug)] @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server-side response storage (omits previous_response_id and controls store flag) + #[arg(long = "disable-response-storage", action = ArgAction::SetTrue, default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..e9a8518d7c 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -32,6 +32,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let Cli { skip_git_repo_check, + disable_response_storage, model, images, prompt, @@ -51,7 +52,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + codex_wrapper::init_codex(approval_policy, sandbox_policy, disable_response_storage, model).await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..7f3cd4a414 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + // by default, use server-side storage + disable_response_storage: false, }, }; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..d8228a131a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -63,8 +63,9 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { + // Initialize session; storage enabled by default let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { + match init_codex(approval_policy, sandbox_policy, false, model).await { Ok(vals) => vals, Err(e) => { // TODO(mbolin): This error needs to be surfaced to the user. From 14ec2086aacaf56536180d338e442216eda38fc0 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 69/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 14 +++- codex-rs/core/src/codex.rs | 74 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 51 ++++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 6 +- codex-rs/exec/src/lib.rs | 3 +- codex-rs/repl/src/lib.rs | 2 + codex-rs/tui/src/chatwidget.rs | 3 +- 14 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..b61741f303 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -51,13 +56,15 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + store: bool, stream: bool, } @@ -152,6 +159,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..5af1080d2f 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,10 +598,23 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut turn_input: Vec = + if let Some(transcript) = &sess.state.lock().unwrap().zdr_transcript { + // If we are using ZDR, we need to send the transcript with every turn. + transcript.contents() + } else { + Vec::new() + }; + + turn_input.extend(pending_response_input.drain(..).map(ResponseItem::from)); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + turn_input.extend(pending_input); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { @@ -598,7 +622,17 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { debug!("Turn completed"); break; } - turn_input = turn_output; + + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + let num_added = transcript.record_items(turn_output.iter().map(|i| &i.item)); + if num_added == 0 { + debug!("Turn completed"); + break; + } + } + + pending_response_input = + turn_output.into_iter().filter_map(|i| i.response).collect(); } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +658,27 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + ( + state.previous_response_id.clone(), + state.zdr_transcript.is_none(), + ) }; let instructions = match prev_id { Some(_) => None, None => sess.instructions.clone(), }; + // Build prompt payload, including store flag let prompt = Prompt { input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +716,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +743,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..451b4f748b --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,51 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items<'a, I>(&mut self, items: I) -> usize + where + I: IntoIterator, + { + let mut count = 0; + for item in items { + if is_api_message(item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + count += 1; + } + } + count + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => { + role.as_str() != "system" && role.as_str() != "assistant" + } + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..938de29d23 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, ArgAction}; use std::path::PathBuf; #[derive(Parser, Debug)] @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server-side response storage (omits previous_response_id and controls store flag) + #[arg(long = "disable-response-storage", action = ArgAction::SetTrue, default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..e9a8518d7c 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -32,6 +32,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let Cli { skip_git_repo_check, + disable_response_storage, model, images, prompt, @@ -51,7 +52,7 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + codex_wrapper::init_codex(approval_policy, sandbox_policy, disable_response_storage, model).await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..7f3cd4a414 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + // by default, use server-side storage + disable_response_storage: false, }, }; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..d8228a131a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -63,8 +63,9 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { + // Initialize session; storage enabled by default let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { + match init_codex(approval_policy, sandbox_policy, false, model).await { Ok(vals) => vals, Err(e) => { // TODO(mbolin): This error needs to be surfaced to the user. From 34c2daf1b4f93245285cdf6f19f0f8fa9d8f351d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 70/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 14 +++- codex-rs/core/src/codex.rs | 73 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 ++++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 51 ++++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 4 ++ codex-rs/exec/src/lib.rs | 14 ++-- codex-rs/repl/src/cli.rs | 4 ++ codex-rs/repl/src/lib.rs | 1 + codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 26 +++++--- codex-rs/tui/src/cli.rs | 4 ++ codex-rs/tui/src/lib.rs | 2 + 18 files changed, 186 insertions(+), 29 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..b61741f303 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -51,13 +56,15 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + store: bool, stream: bool, } @@ -152,6 +159,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..e78a89324c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,10 +598,23 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut turn_input: Vec = + if let Some(transcript) = &sess.state.lock().unwrap().zdr_transcript { + // If we are using ZDR, we need to send the transcript with every turn. + transcript.contents() + } else { + Vec::new() + }; + + turn_input.extend(pending_response_input.drain(..).map(ResponseItem::from)); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + turn_input.extend(pending_input); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { @@ -598,7 +622,17 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { debug!("Turn completed"); break; } - turn_input = turn_output; + + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + let num_added = transcript.record_items(turn_output.iter().map(|i| &i.item)); + if num_added == 0 { + debug!("Turn completed"); + break; + } + } + + pending_response_input = + turn_output.into_iter().filter_map(|i| i.response).collect(); } Err(e) => { info!("Turn error: {e:#}"); @@ -624,11 +658,15 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + ( + state.previous_response_id.clone(), + state.zdr_transcript.is_none(), + ) }; let instructions = match prev_id { @@ -639,6 +677,7 @@ async fn run_turn( input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +715,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +742,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..451b4f748b --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,51 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items<'a, I>(&mut self, items: I) -> usize + where + I: IntoIterator, + { + let mut count = 0; + for item in items { + if is_api_message(item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + count += 1; + } + } + count + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => { + role.as_str() != "system" && role.as_str() != "assistant" + } + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..299e85879d 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..ab7d735e0f 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -31,9 +31,10 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { .try_init(); let Cli { - skip_git_repo_check, - model, images, + model, + skip_git_repo_check, + disable_response_storage, prompt, .. } = cli; @@ -50,8 +51,13 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // likely come from a new --execution-policy arg. let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; - let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index bb83046d3c..4de42a76de 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -50,6 +50,10 @@ pub struct Cli { #[arg(long, action = ArgAction::SetTrue, default_value_t = false)] pub allow_no_git_exec: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Record submissions into file as JSON #[arg(short = 'S', long)] pub record_submissions: Option, diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..0f9c47e49b 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,7 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + disable_response_storage: cli.disable_response_storage, }, }; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..3b6df1df21 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,6 +37,7 @@ impl App<'_> { show_git_warning: bool, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (app_event_tx, app_event_rx) = channel(); @@ -70,6 +71,7 @@ impl App<'_> { initial_prompt.clone(), initial_images, model, + disable_response_storage, ); let app_state = if show_git_warning { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..edcfc40038 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -52,6 +52,7 @@ impl ChatWidget<'_> { initial_prompt: Option, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -63,15 +64,22 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { - Ok(vals) => vals, - Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. - tracing::error!("failed to initialize codex: {e}"); - return; - } - }; + // Initialize session; storage enabled by default + let (codex, session_event, _ctrl_c) = match init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await + { + Ok(vals) => vals, + Err(e) => { + // TODO(mbolin): This error needs to be surfaced to the user. + tracing::error!("failed to initialize codex: {e}"); + return; + } + }; // Forward the captured `SessionInitialized` event that was consumed // inside `init_codex()` so it can be rendered in the UI. diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index fa764d1ab3..db25ad2b3c 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -31,6 +31,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Convenience alias for low-friction sandboxed automatic execution (-a on-failure, -s network-and-file-write-restricted) #[arg(long = "full-auto", default_value_t = true)] pub full_auto: bool, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..527668ad24 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -106,6 +106,7 @@ fn run_ratatui_app( approval_policy, sandbox_policy: sandbox, model, + disable_response_storage, .. } = cli; @@ -119,6 +120,7 @@ fn run_ratatui_app( show_git_warning, images, model, + disable_response_storage, ); // Bridge log receiver into the AppEvent channel so latest log lines update the UI. From b05ea21b1773f6374c9e8cc4c04d2588082046a3 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 24 Apr 2025 23:54:20 -0700 Subject: [PATCH 71/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 17 ++++- codex-rs/core/src/codex.rs | 82 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 48 ++++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 4 + codex-rs/exec/src/lib.rs | 14 +++- codex-rs/repl/src/cli.rs | 4 + codex-rs/repl/src/lib.rs | 1 + codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 26 ++++--- codex-rs/tui/src/cli.rs | 4 + codex-rs/tui/src/lib.rs | 2 + 18 files changed, 194 insertions(+), 30 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..73d10ea4c9 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -51,13 +56,18 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, + // we code defensively to avoid this case, but perhaps we should use a + // separate enum for serialization. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + /// true when using the Responses API. + store: bool, stream: bool, } @@ -152,6 +162,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..40e142c947 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,18 +598,48 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut turn_input: Vec = + if let Some(transcript) = &sess.state.lock().unwrap().zdr_transcript { + // If we are using ZDR, we need to send the transcript with every turn. + transcript.contents() + } else { + Vec::new() + }; + + turn_input.extend(pending_response_input.drain(..).map(ResponseItem::from)); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + turn_input.extend(pending_input); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { - if turn_output.is_empty() { + let (items, responses): (Vec<_>, Vec<_>) = turn_output + .into_iter() + .map(|p| (p.item, p.response)) + .unzip(); + let responses = responses + .into_iter() + .flatten() + .collect::>(); + + // Only attempt to take the lock if there is something to record. + if !items.is_empty() { + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + transcript.record_items(items); + } + } + + if responses.is_empty() { debug!("Turn completed"); break; } - turn_input = turn_output; + + pending_response_input = responses; } Err(e) => { info!("Turn error: {e:#}"); @@ -624,11 +665,15 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + ( + state.previous_response_id.clone(), + state.zdr_transcript.is_none(), + ) }; let instructions = match prev_id { @@ -639,6 +684,7 @@ async fn run_turn( input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +722,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +749,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..ca9adc0ff3 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,48 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items(&mut self, items: I) + where + I: IntoIterator, + { + for item in items { + if is_api_message(&item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + } + } + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => { + role.as_str() != "system" && role.as_str() != "assistant" + } + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..299e85879d 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..ab7d735e0f 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -31,9 +31,10 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { .try_init(); let Cli { - skip_git_repo_check, - model, images, + model, + skip_git_repo_check, + disable_response_storage, prompt, .. } = cli; @@ -50,8 +51,13 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // likely come from a new --execution-policy arg. let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; - let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index bb83046d3c..4de42a76de 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -50,6 +50,10 @@ pub struct Cli { #[arg(long, action = ArgAction::SetTrue, default_value_t = false)] pub allow_no_git_exec: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Record submissions into file as JSON #[arg(short = 'S', long)] pub record_submissions: Option, diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..0f9c47e49b 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,7 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + disable_response_storage: cli.disable_response_storage, }, }; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..3b6df1df21 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,6 +37,7 @@ impl App<'_> { show_git_warning: bool, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (app_event_tx, app_event_rx) = channel(); @@ -70,6 +71,7 @@ impl App<'_> { initial_prompt.clone(), initial_images, model, + disable_response_storage, ); let app_state = if show_git_warning { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..edcfc40038 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -52,6 +52,7 @@ impl ChatWidget<'_> { initial_prompt: Option, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -63,15 +64,22 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { - Ok(vals) => vals, - Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. - tracing::error!("failed to initialize codex: {e}"); - return; - } - }; + // Initialize session; storage enabled by default + let (codex, session_event, _ctrl_c) = match init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await + { + Ok(vals) => vals, + Err(e) => { + // TODO(mbolin): This error needs to be surfaced to the user. + tracing::error!("failed to initialize codex: {e}"); + return; + } + }; // Forward the captured `SessionInitialized` event that was consumed // inside `init_codex()` so it can be rendered in the UI. diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index fa764d1ab3..db25ad2b3c 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -31,6 +31,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Convenience alias for low-friction sandboxed automatic execution (-a on-failure, -s network-and-file-write-restricted) #[arg(long = "full-auto", default_value_t = true)] pub full_auto: bool, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..527668ad24 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -106,6 +106,7 @@ fn run_ratatui_app( approval_policy, sandbox_policy: sandbox, model, + disable_response_storage, .. } = cli; @@ -119,6 +120,7 @@ fn run_ratatui_app( show_git_warning, images, model, + disable_response_storage, ); // Bridge log receiver into the AppEvent channel so latest log lines update the UI. From 2e1904f308fb2db7b9dfaa4f8d0cd210a71952eb Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 11:44:27 -0700 Subject: [PATCH 72/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 17 +++- codex-rs/core/src/codex.rs | 99 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 48 ++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 4 + codex-rs/exec/src/lib.rs | 14 ++- codex-rs/repl/src/cli.rs | 4 + codex-rs/repl/src/lib.rs | 1 + codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 26 ++++-- codex-rs/tui/src/cli.rs | 4 + codex-rs/tui/src/lib.rs | 2 + 18 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57f593a884..73d10ea4c9 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -29,15 +29,20 @@ use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::flags::OPENAI_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -51,13 +56,18 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, + // we code defensively to avoid this case, but perhaps we should use a + // separate enum for serialization. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + /// true when using the Responses API. + store: bool, stream: bool, } @@ -152,6 +162,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..0d17c8e47e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,18 +598,54 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut net_new_turn_input = pending_response_input + .drain(..) + .map(ResponseItem::from) + .collect::>(); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + net_new_turn_input.extend(pending_input); + + let turn_input: Vec = + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + // If we are using ZDR, we need to send the transcript with every turn. + let mut full_transcript = transcript.contents(); + full_transcript.extend(net_new_turn_input.clone()); + transcript.record_items(net_new_turn_input); + full_transcript + } else { + net_new_turn_input + }; match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { - if turn_output.is_empty() { + let (items, responses): (Vec<_>, Vec<_>) = turn_output + .into_iter() + .map(|p| (p.item, p.response)) + .unzip(); + let responses = responses + .into_iter() + .flatten() + .collect::>(); + + // Only attempt to take the lock if there is something to record. + if !items.is_empty() { + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + transcript.record_items(items); + } + } + + if responses.is_empty() { debug!("Turn completed"); break; } - turn_input = turn_output; + + pending_response_input = responses; } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +671,31 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store, is_first_turn) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + let is_first_turn = state.previous_response_id.is_none(); + if state.zdr_transcript.is_some() { + // When using ZDR, the Reponses API may send previous_response_id + // back, but trying to use it results in a 400. + (None, true, is_first_turn) + } else { + (state.previous_response_id.clone(), false, is_first_turn) + } }; - let instructions = match prev_id { - Some(_) => None, - None => sess.instructions.clone(), + let instructions = if is_first_turn { + sess.instructions.clone() + } else { + None }; let prompt = Prompt { input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +733,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +760,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..ca9adc0ff3 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,48 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items(&mut self, items: I) + where + I: IntoIterator, + { + for item in items { + if is_api_message(&item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + } + } + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => { + role.as_str() != "system" && role.as_str() != "assistant" + } + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..299e85879d 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..ab7d735e0f 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -31,9 +31,10 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { .try_init(); let Cli { - skip_git_repo_check, - model, images, + model, + skip_git_repo_check, + disable_response_storage, prompt, .. } = cli; @@ -50,8 +51,13 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // likely come from a new --execution-policy arg. let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; - let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index bb83046d3c..4de42a76de 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -50,6 +50,10 @@ pub struct Cli { #[arg(long, action = ArgAction::SetTrue, default_value_t = false)] pub allow_no_git_exec: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Record submissions into file as JSON #[arg(short = 'S', long)] pub record_submissions: Option, diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..0f9c47e49b 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,7 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + disable_response_storage: cli.disable_response_storage, }, }; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..3b6df1df21 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,6 +37,7 @@ impl App<'_> { show_git_warning: bool, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (app_event_tx, app_event_rx) = channel(); @@ -70,6 +71,7 @@ impl App<'_> { initial_prompt.clone(), initial_images, model, + disable_response_storage, ); let app_state = if show_git_warning { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..edcfc40038 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -52,6 +52,7 @@ impl ChatWidget<'_> { initial_prompt: Option, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -63,15 +64,22 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { - Ok(vals) => vals, - Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. - tracing::error!("failed to initialize codex: {e}"); - return; - } - }; + // Initialize session; storage enabled by default + let (codex, session_event, _ctrl_c) = match init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await + { + Ok(vals) => vals, + Err(e) => { + // TODO(mbolin): This error needs to be surfaced to the user. + tracing::error!("failed to initialize codex: {e}"); + return; + } + }; // Forward the captured `SessionInitialized` event that was consumed // inside `init_codex()` so it can be rendered in the UI. diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index fa764d1ab3..db25ad2b3c 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -31,6 +31,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Convenience alias for low-friction sandboxed automatic execution (-a on-failure, -s network-and-file-write-restricted) #[arg(long = "full-auto", default_value_t = true)] pub full_auto: bool, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..527668ad24 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -106,6 +106,7 @@ fn run_ratatui_app( approval_policy, sandbox_policy: sandbox, model, + disable_response_storage, .. } = cli; @@ -119,6 +120,7 @@ fn run_ratatui_app( show_git_warning, images, model, + disable_response_storage, ); // Bridge log receiver into the AppEvent channel so latest log lines update the UI. From 285c335951b3205cfb79c24679aa16f9d0bc0299 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 11:48:21 -0700 Subject: [PATCH 73/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 17 +++- codex-rs/core/src/codex.rs | 99 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 48 ++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 4 + codex-rs/exec/src/lib.rs | 14 ++- codex-rs/repl/src/cli.rs | 4 + codex-rs/repl/src/lib.rs | 1 + codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 26 ++++-- codex-rs/tui/src/cli.rs | 4 + codex-rs/tui/src/lib.rs | 2 + 18 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 51e8b8e844..10ec0b9780 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -28,15 +28,20 @@ use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -50,13 +55,18 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, + // we code defensively to avoid this case, but perhaps we should use a + // separate enum for serialization. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + /// true when using the Responses API. + store: bool, stream: bool, } @@ -151,6 +161,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..0d17c8e47e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,18 +598,54 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut net_new_turn_input = pending_response_input + .drain(..) + .map(ResponseItem::from) + .collect::>(); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + net_new_turn_input.extend(pending_input); + + let turn_input: Vec = + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + // If we are using ZDR, we need to send the transcript with every turn. + let mut full_transcript = transcript.contents(); + full_transcript.extend(net_new_turn_input.clone()); + transcript.record_items(net_new_turn_input); + full_transcript + } else { + net_new_turn_input + }; match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { - if turn_output.is_empty() { + let (items, responses): (Vec<_>, Vec<_>) = turn_output + .into_iter() + .map(|p| (p.item, p.response)) + .unzip(); + let responses = responses + .into_iter() + .flatten() + .collect::>(); + + // Only attempt to take the lock if there is something to record. + if !items.is_empty() { + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + transcript.record_items(items); + } + } + + if responses.is_empty() { debug!("Turn completed"); break; } - turn_input = turn_output; + + pending_response_input = responses; } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +671,31 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store, is_first_turn) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + let is_first_turn = state.previous_response_id.is_none(); + if state.zdr_transcript.is_some() { + // When using ZDR, the Reponses API may send previous_response_id + // back, but trying to use it results in a 400. + (None, true, is_first_turn) + } else { + (state.previous_response_id.clone(), false, is_first_turn) + } }; - let instructions = match prev_id { - Some(_) => None, - None => sess.instructions.clone(), + let instructions = if is_first_turn { + sess.instructions.clone() + } else { + None }; let prompt = Prompt { input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +733,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +760,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..ca9adc0ff3 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,48 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items(&mut self, items: I) + where + I: IntoIterator, + { + for item in items { + if is_api_message(&item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + } + } + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => { + role.as_str() != "system" && role.as_str() != "assistant" + } + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..299e85879d 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..ab7d735e0f 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -31,9 +31,10 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { .try_init(); let Cli { - skip_git_repo_check, - model, images, + model, + skip_git_repo_check, + disable_response_storage, prompt, .. } = cli; @@ -50,8 +51,13 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // likely come from a new --execution-policy arg. let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; - let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index bb83046d3c..4de42a76de 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -50,6 +50,10 @@ pub struct Cli { #[arg(long, action = ArgAction::SetTrue, default_value_t = false)] pub allow_no_git_exec: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Record submissions into file as JSON #[arg(short = 'S', long)] pub record_submissions: Option, diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..0f9c47e49b 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,7 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + disable_response_storage: cli.disable_response_storage, }, }; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..3b6df1df21 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,6 +37,7 @@ impl App<'_> { show_git_warning: bool, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (app_event_tx, app_event_rx) = channel(); @@ -70,6 +71,7 @@ impl App<'_> { initial_prompt.clone(), initial_images, model, + disable_response_storage, ); let app_state = if show_git_warning { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..edcfc40038 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -52,6 +52,7 @@ impl ChatWidget<'_> { initial_prompt: Option, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -63,15 +64,22 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { - Ok(vals) => vals, - Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. - tracing::error!("failed to initialize codex: {e}"); - return; - } - }; + // Initialize session; storage enabled by default + let (codex, session_event, _ctrl_c) = match init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await + { + Ok(vals) => vals, + Err(e) => { + // TODO(mbolin): This error needs to be surfaced to the user. + tracing::error!("failed to initialize codex: {e}"); + return; + } + }; // Forward the captured `SessionInitialized` event that was consumed // inside `init_codex()` so it can be rendered in the UI. diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index fa764d1ab3..db25ad2b3c 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -31,6 +31,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Convenience alias for low-friction sandboxed automatic execution (-a on-failure, -s network-and-file-write-restricted) #[arg(long = "full-auto", default_value_t = true)] pub full_auto: bool, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..527668ad24 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -106,6 +106,7 @@ fn run_ratatui_app( approval_policy, sandbox_policy: sandbox, model, + disable_response_storage, .. } = cli; @@ -119,6 +120,7 @@ fn run_ratatui_app( show_git_warning, images, model, + disable_response_storage, ); // Bridge log receiver into the AppEvent channel so latest log lines update the UI. From 42209b172c2cd33a5dc0e6c1fa111df2b297438e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 11:48:21 -0700 Subject: [PATCH 74/84] feat: add ZDR support to Rust implementation --- codex-rs/core/src/client.rs | 17 +++- codex-rs/core/src/codex.rs | 99 +++++++++++++++++---- codex-rs/core/src/codex_wrapper.rs | 2 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/models.rs | 11 +++ codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/zdr_transcript.rs | 46 ++++++++++ codex-rs/core/tests/live_agent.rs | 1 + codex-rs/core/tests/previous_response_id.rs | 1 + codex-rs/core/tests/stream_no_completed.rs | 1 + codex-rs/exec/src/cli.rs | 4 + codex-rs/exec/src/lib.rs | 14 ++- codex-rs/repl/src/cli.rs | 4 + codex-rs/repl/src/lib.rs | 1 + codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 26 ++++-- codex-rs/tui/src/cli.rs | 4 + codex-rs/tui/src/lib.rs | 2 + 18 files changed, 206 insertions(+), 33 deletions(-) create mode 100644 codex-rs/core/src/zdr_transcript.rs diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 51e8b8e844..10ec0b9780 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -28,15 +28,20 @@ use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::flags::OPENAI_API_BASE; use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; -use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::util::backoff; +/// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { - pub input: Vec, + /// Conversation context input items. + pub input: Vec, + /// Optional previous response ID (when storage is enabled). pub prev_id: Option, + /// Optional initial instructions (only sent on first turn). pub instructions: Option, + /// Whether to store response on server side (disable_response_storage = !store). + pub store: bool, } #[derive(Debug)] @@ -50,13 +55,18 @@ struct Payload<'a> { model: &'a str, #[serde(skip_serializing_if = "Option::is_none")] instructions: Option<&'a String>, - input: &'a Vec, + // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, + // we code defensively to avoid this case, but perhaps we should use a + // separate enum for serialization. + input: &'a Vec, tools: &'a [Tool], tool_choice: &'static str, parallel_tool_calls: bool, reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] previous_response_id: Option, + /// true when using the Responses API. + store: bool, stream: bool, } @@ -151,6 +161,7 @@ impl ModelClient { generate_summary: None, }), previous_response_id: prompt.prev_id.clone(), + store: prompt.store, stream: true, }; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e57d3bbf07..0d17c8e47e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use crate::safety::assess_command_safety; use crate::safety::assess_patch_safety; use crate::safety::SafetyCheck; use crate::util::backoff; +use crate::zdr_transcript::ZdrTranscript; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -214,6 +215,7 @@ struct State { previous_response_id: Option, pending_approvals: HashMap>, pending_input: Vec, + zdr_transcript: Option, } impl Session { @@ -399,6 +401,7 @@ impl State { Self { approved_commands: self.approved_commands.clone(), previous_response_id: self.previous_response_id.clone(), + zdr_transcript: self.zdr_transcript.clone(), ..Default::default() } } @@ -489,6 +492,7 @@ async fn submission_loop( instructions, approval_policy, sandbox_policy, + disable_response_storage, } => { let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); @@ -500,7 +504,14 @@ async fn submission_loop( sess.abort(); sess.state.lock().unwrap().partial_clone() } - None => State::default(), + None => State { + zdr_transcript: if disable_response_storage { + Some(ZdrTranscript::new()) + } else { + None + }, + ..Default::default() + }, }; // update session @@ -587,18 +598,54 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut turn_input = vec![ResponseInputItem::from(input)]; + let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; loop { - let pending_input = sess.get_pending_input(); - turn_input.splice(0..0, pending_input); + let mut net_new_turn_input = pending_response_input + .drain(..) + .map(ResponseItem::from) + .collect::>(); + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); + net_new_turn_input.extend(pending_input); + + let turn_input: Vec = + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + // If we are using ZDR, we need to send the transcript with every turn. + let mut full_transcript = transcript.contents(); + full_transcript.extend(net_new_turn_input.clone()); + transcript.record_items(net_new_turn_input); + full_transcript + } else { + net_new_turn_input + }; match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { - if turn_output.is_empty() { + let (items, responses): (Vec<_>, Vec<_>) = turn_output + .into_iter() + .map(|p| (p.item, p.response)) + .unzip(); + let responses = responses + .into_iter() + .flatten() + .collect::>(); + + // Only attempt to take the lock if there is something to record. + if !items.is_empty() { + if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { + transcript.record_items(items); + } + } + + if responses.is_empty() { debug!("Turn completed"); break; } - turn_input = turn_output; + + pending_response_input = responses; } Err(e) => { info!("Turn error: {e:#}"); @@ -624,21 +671,31 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, sub_id: String, - input: Vec, -) -> CodexResult> { - let prev_id = { + input: Vec, +) -> CodexResult> { + // Decide whether to use server-side storage (previous_response_id) or disable it + let (prev_id, store, is_first_turn) = { let state = sess.state.lock().unwrap(); - state.previous_response_id.clone() + let is_first_turn = state.previous_response_id.is_none(); + if state.zdr_transcript.is_some() { + // When using ZDR, the Reponses API may send previous_response_id + // back, but trying to use it results in a 400. + (None, true, is_first_turn) + } else { + (state.previous_response_id.clone(), false, is_first_turn) + } }; - let instructions = match prev_id { - Some(_) => None, - None => sess.instructions.clone(), + let instructions = if is_first_turn { + sess.instructions.clone() + } else { + None }; let prompt = Prompt { input, prev_id, instructions, + store, }; let mut retries = 0; @@ -676,11 +733,20 @@ async fn run_turn( } } +/// When the model is prompted, it returns a stream of events. Some of these +/// events map to a `ResponseItem`. A `ResponseItem` may need to be +/// "handled" such that it produces a `ResponseInputItem` that needs to be +/// sent back to the model on the next turn. +struct ProcessedResponseItem { + item: ResponseItem, + response: Option, +} + async fn try_run_turn( sess: &Session, sub_id: &str, prompt: &Prompt, -) -> CodexResult> { +) -> CodexResult> { let mut stream = sess.client.clone().stream(prompt).await?; // Buffer all the incoming messages from the stream first, then execute them. @@ -694,9 +760,8 @@ async fn try_run_turn( for event in input { match event { ResponseEvent::OutputItemDone(item) => { - if let Some(item) = handle_response_item(sess, sub_id, item).await? { - output.push(item); - } + let response = handle_response_item(sess, sub_id, item.clone()).await?; + output.push(ProcessedResponseItem { item, response }); } ResponseEvent::Completed { response_id } => { let mut state = sess.state.lock().unwrap(); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 426b5373c5..8d19683ffa 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -21,6 +21,7 @@ use tracing::debug; pub async fn init_codex( approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, + disable_response_storage: bool, model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); @@ -33,6 +34,7 @@ pub async fn init_codex( instructions: config.instructions, approval_policy, sandbox_policy, + disable_response_storage, }) .await?; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 7d3309152c..d517e68824 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod models; pub mod protocol; mod safety; pub mod util; +mod zdr_transcript; pub use codex::Codex; diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 551ac31815..2665e8c17b 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -56,6 +56,17 @@ pub enum ResponseItem { Other, } +impl From for ResponseItem { + fn from(item: ResponseInputItem) -> Self { + match item { + ResponseInputItem::Message { role, content } => Self::Message { role, content }, + ResponseInputItem::FunctionCallOutput { call_id, output } => { + Self::FunctionCallOutput { call_id, output } + } + } + } +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 42c8478e6b..96c4ea4832 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -33,6 +33,9 @@ pub enum Op { approval_policy: AskForApproval, /// How to sandbox commands executed in the system sandbox_policy: SandboxPolicy, + /// Disable server-side response storage (send full context each request) + #[serde(default)] + disable_response_storage: bool, }, /// Abort current task. diff --git a/codex-rs/core/src/zdr_transcript.rs b/codex-rs/core/src/zdr_transcript.rs new file mode 100644 index 0000000000..25fdc5a679 --- /dev/null +++ b/codex-rs/core/src/zdr_transcript.rs @@ -0,0 +1,46 @@ +use crate::models::ResponseItem; + +/// Transcript that needs to be maintained for ZDR clients for which +/// previous_response_id is not available, so we must include the transcript +/// with every API call. This must include each `function_call` and its +/// corresponding `function_call_output`. +#[derive(Debug, Clone)] +pub(crate) struct ZdrTranscript { + /// The oldest items are at the beginning of the vector. + items: Vec, +} + +impl ZdrTranscript { + pub(crate) fn new() -> Self { + Self { items: Vec::new() } + } + + /// Returns a clone of the contents in the transcript. + pub(crate) fn contents(&self) -> Vec { + self.items.clone() + } + + /// `items` is ordered from oldest to newest. + pub(crate) fn record_items(&mut self, items: I) + where + I: IntoIterator, + { + for item in items { + if is_api_message(&item) { + // Note agent-loop.ts also does filtering on some of the fields. + self.items.push(item.clone()); + } + } + } +} + +/// Anything that is not a system message or "reasoning" message is considered +/// an API message. +fn is_api_message(message: &ResponseItem) -> bool { + match message { + ResponseItem::Message { role, .. } => role.as_str() != "system", + ResponseItem::FunctionCall { .. } => true, + ResponseItem::FunctionCallOutput { .. } => true, + _ => false, + } +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 6562654c23..823cd73a01 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -55,6 +55,7 @@ async fn spawn_codex() -> Codex { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index 56fa9a6c0b..de1309e856 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -95,6 +95,7 @@ async fn keeps_previous_response_id_between_tasks() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index da0cfb276b..c732a5fdbb 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -78,6 +78,7 @@ async fn retries_on_early_close() { instructions: None, approval_policy: AskForApproval::OnFailure, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, + disable_response_storage: false, }, }) .await diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index a934aba003..299e85879d 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -16,6 +16,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Initial instructions for the agent. pub prompt: Option, } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c22b6bd694..ab7d735e0f 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -31,9 +31,10 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { .try_init(); let Cli { - skip_git_repo_check, - model, images, + model, + skip_git_repo_check, + disable_response_storage, prompt, .. } = cli; @@ -50,8 +51,13 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // likely come from a new --execution-policy arg. let approval_policy = AskForApproval::Never; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; - let (codex_wrapper, event, ctrl_c) = - codex_wrapper::init_codex(approval_policy, sandbox_policy, model).await?; + let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await?; let codex = Arc::new(codex_wrapper); info!("Codex initialized with event: {event:?}"); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index bb83046d3c..4de42a76de 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -50,6 +50,10 @@ pub struct Cli { #[arg(long, action = ArgAction::SetTrue, default_value_t = false)] pub allow_no_git_exec: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Record submissions into file as JSON #[arg(short = 'S', long)] pub record_submissions: Option, diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 2266718ed9..0f9c47e49b 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -97,6 +97,7 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R instructions: cfg.instructions, approval_policy: cli.approval_policy.into(), sandbox_policy: cli.sandbox_policy.into(), + disable_response_storage: cli.disable_response_storage, }, }; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9aba46ec8f..3b6df1df21 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,6 +37,7 @@ impl App<'_> { show_git_warning: bool, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (app_event_tx, app_event_rx) = channel(); @@ -70,6 +71,7 @@ impl App<'_> { initial_prompt.clone(), initial_images, model, + disable_response_storage, ); let app_state = if show_git_warning { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 149cea42c4..edcfc40038 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -52,6 +52,7 @@ impl ChatWidget<'_> { initial_prompt: Option, initial_images: Vec, model: Option, + disable_response_storage: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -63,15 +64,22 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - let (codex, session_event, _ctrl_c) = - match init_codex(approval_policy, sandbox_policy, model).await { - Ok(vals) => vals, - Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. - tracing::error!("failed to initialize codex: {e}"); - return; - } - }; + // Initialize session; storage enabled by default + let (codex, session_event, _ctrl_c) = match init_codex( + approval_policy, + sandbox_policy, + disable_response_storage, + model, + ) + .await + { + Ok(vals) => vals, + Err(e) => { + // TODO(mbolin): This error needs to be surfaced to the user. + tracing::error!("failed to initialize codex: {e}"); + return; + } + }; // Forward the captured `SessionInitialized` event that was consumed // inside `init_codex()` so it can be rendered in the UI. diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index fa764d1ab3..db25ad2b3c 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -31,6 +31,10 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, + /// Disable server‑side response storage (sends the full conversation context with every request) + #[arg(long = "disable-response-storage", default_value_t = false)] + pub disable_response_storage: bool, + /// Convenience alias for low-friction sandboxed automatic execution (-a on-failure, -s network-and-file-write-restricted) #[arg(long = "full-auto", default_value_t = true)] pub full_auto: bool, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 598d3eaf1b..527668ad24 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -106,6 +106,7 @@ fn run_ratatui_app( approval_policy, sandbox_policy: sandbox, model, + disable_response_storage, .. } = cli; @@ -119,6 +120,7 @@ fn run_ratatui_app( show_git_warning, images, model, + disable_response_storage, ); // Bridge log receiver into the AppEvent channel so latest log lines update the UI. From 7ab4366709e301b817d61923f5b47dbea6998f38 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 13:32:22 -0700 Subject: [PATCH 75/84] fix: flipped the sense of Prompt.store in #642 --- codex-rs/core/src/codex.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 0d17c8e47e..a23c11cde5 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -677,13 +677,15 @@ async fn run_turn( let (prev_id, store, is_first_turn) = { let state = sess.state.lock().unwrap(); let is_first_turn = state.previous_response_id.is_none(); - if state.zdr_transcript.is_some() { + let store = state.zdr_transcript.is_none(); + let prev_id = if store { + state.previous_response_id.clone() + } else { // When using ZDR, the Reponses API may send previous_response_id // back, but trying to use it results in a 400. - (None, true, is_first_turn) - } else { - (state.previous_response_id.clone(), false, is_first_turn) - } + None + }; + (prev_id, store, is_first_turn) }; let instructions = if is_first_turn { From 8c09afb5af5fd06aa86f8efa2ca0e7dbd86317cc Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 13:47:07 -0700 Subject: [PATCH 76/84] ci: build Rust on Windows as part of CI to ensure we guard platform-specific code appropriately --- .github/workflows/rust-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 7e200960d9..98c98d66bd 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -41,6 +41,8 @@ jobs: strategy: fail-fast: false matrix: + # Note: While Codex CLI does not support Windows today, we include + # Windows in CI to ensure the code at least builds there. include: - runner: macos-14 target: aarch64-apple-darwin @@ -50,6 +52,8 @@ jobs: target: x86_64-unknown-linux-musl - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + - runner: windows-latest + target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v4 From bd74cccddc891d9dd2d63cf6fdcac77c13541ddd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 13:47:07 -0700 Subject: [PATCH 77/84] ci: build Rust on Windows as part of CI to ensure we guard platform-specific code appropriately --- .github/workflows/rust-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 7e200960d9..eb399cbce3 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -5,6 +5,7 @@ on: - main paths: - "codex-rs/**" + - ".github/**" push: branches: - main @@ -41,6 +42,8 @@ jobs: strategy: fail-fast: false matrix: + # Note: While Codex CLI does not support Windows today, we include + # Windows in CI to ensure the code at least builds there. include: - runner: macos-14 target: aarch64-apple-darwin @@ -50,6 +53,8 @@ jobs: target: x86_64-unknown-linux-musl - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + - runner: windows-latest + target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v4 From 9a19126a7397430ae8056155c1bc05c9910536fa Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 14:06:05 -0700 Subject: [PATCH 78/84] fix: write logs to ~/.codex/log instead of /tmp --- codex-rs/core/src/config.rs | 31 +++++++++++++++++++++++++++---- codex-rs/tui/src/lib.rs | 26 +++++++++++++++++++------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index c094de5436..b5574ceade 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use dirs::home_dir; use serde::Deserialize; @@ -28,15 +30,36 @@ impl Config { } fn load_from_toml() -> Option { - let mut p = home_dir()?; - p.push(".codex/config.toml"); + let mut p = codex_dir().ok()?; + p.push("config.toml"); let contents = std::fs::read_to_string(&p).ok()?; toml::from_str(&contents).ok() } fn load_instructions() -> Option { - let mut p = home_dir()?; - p.push(".codex/instructions.md"); + let mut p = codex_dir().ok()?; + p.push("instructions.md"); std::fs::read_to_string(&p).ok() } } + +/// Returns the path to the Codex configuration directory, which is `~/.codex`. +/// Does not verify that the directory exists. +pub fn codex_dir() -> std::io::Result { + let mut p = home_dir().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "Could not find home directory", + ) + })?; + p.push(".codex"); + Ok(p) +} + +/// Returns the path to the folder where Codex logs are stored. Does not verify +/// that the directory exists. +pub fn log_dir() -> std::io::Result { + let mut p = codex_dir()?; + p.push("log"); + Ok(p) +} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 4a063de084..d0f5f664a6 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -31,19 +31,31 @@ pub use cli::Cli; pub fn run_main(cli: Cli) -> std::io::Result<()> { assert_env_var_set(); + let log_dir = codex_core::config::log_dir()?; + std::fs::create_dir_all(&log_dir)?; // Open (or create) your log file, appending to it. - let file = OpenOptions::new() - .create(true) - .append(true) - .open("/tmp/codex-rs.log")?; + let mut log_file_opts = OpenOptions::new(); + log_file_opts.create(true).append(true); + + // Ensure the file is only readable and writable by the current user. + // Doing the equivalent to `chmod 600` on Windows is quite a bit more code + // and requires the Windows API crates, so we can reconsider that when + // Codex CLI is officially supported on Windows. + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + log_file_opts.mode(0o600); + } + + let log_file = log_file_opts.open(log_dir.join("codex-tui.log"))?; // Wrap file in non‑blocking writer. - let (non_blocking, _guard) = non_blocking(file); + let (non_blocking, _guard) = non_blocking(log_file); - // use RUST_LOG env var, default to trace for codex crates. + // use RUST_LOG env var, default to info for codex crates. let env_filter = || { EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("codex=trace,codex_tui=trace")) + .unwrap_or_else(|_| EnvFilter::new("codex_core=info,codex_tui=info")) }; // Build layered subscriber: From a07986e5185057c48bbecaba8cba25a3519798c2 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 14:11:47 -0700 Subject: [PATCH 79/84] fix: remove dependency on expanduser crate --- codex-rs/Cargo.lock | 118 +------------------------------------ codex-rs/core/Cargo.toml | 1 - codex-rs/core/src/codex.rs | 22 +++---- codex-rs/repl/src/cli.rs | 4 +- 4 files changed, 11 insertions(+), 134 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 1f91c0072b..f866ed6beb 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -171,18 +171,6 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "ascii-canvas" version = "3.0.0" @@ -268,12 +256,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.7" @@ -319,17 +301,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" -[[package]] -name = "blake2b_simd" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] - [[package]] name = "bstr" version = "1.12.0" @@ -524,10 +495,9 @@ dependencies = [ "bytes", "clap", "codex-apply-patch", - "dirs 6.0.0", + "dirs", "env-flags", "eventsource-stream", - "expanduser", "fs-err", "futures", "landlock", @@ -684,12 +654,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "convert_case" version = "0.6.0" @@ -890,17 +854,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "dirs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" -dependencies = [ - "libc", - "redox_users 0.3.5", - "winapi", -] - [[package]] name = "dirs" version = "6.0.0" @@ -1132,17 +1085,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "expanduser" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e0b79235da57db6b6c2beed9af6e5de867d63a973ae3e91910ddc33ba40bc0" -dependencies = [ - "dirs 1.0.5", - "lazy_static", - "pwd", -] - [[package]] name = "eyre" version = "0.6.12" @@ -1328,17 +1270,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.16" @@ -2340,7 +2271,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.11", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -2508,16 +2439,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pwd" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" -dependencies = [ - "libc", - "thiserror 1.0.69", -] - [[package]] name = "quote" version = "1.0.40" @@ -2593,12 +2514,6 @@ dependencies = [ "unicode-width 0.2.0", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.5.11" @@ -2608,17 +2523,6 @@ dependencies = [ "bitflags 2.9.0", ] -[[package]] -name = "redox_users" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" -dependencies = [ - "getrandom 0.1.16", - "redox_syscall 0.1.57", - "rust-argon2", -] - [[package]] name = "redox_users" version = "0.4.6" @@ -2765,18 +2669,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rust-argon2" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" -dependencies = [ - "base64 0.13.1", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3908,12 +3800,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 778362d275..daadec7294 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -17,7 +17,6 @@ codex-apply-patch = { path = "../apply-patch" } dirs = "6" env-flags = "0.1.1" eventsource-stream = "0.2.3" -expanduser = "1.2.2" fs-err = "3.1.0" futures = "0.3" mime_guess = "2.0" diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a23c11cde5..cfeb7e401f 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::io::Write; +use std::path::Path; use std::path::PathBuf; use std::process::Command; use std::process::Stdio; @@ -15,7 +16,6 @@ use codex_apply_patch::print_summary; use codex_apply_patch::AffectedPaths; use codex_apply_patch::ApplyPatchFileChange; use codex_apply_patch::MaybeApplyPatchVerified; -use expanduser::expanduser; use fs_err as fs; use futures::prelude::*; use serde::Serialize; @@ -113,23 +113,15 @@ impl CodexBuilder { }) } - pub fn record_submissions(mut self, path: impl AsRef) -> Self { - let path = match expanduser(path.as_ref()) { - Ok(path) => path, - Err(_) => PathBuf::from(path.as_ref()), - }; - debug!("Recording submissions to {}", path.display()); - self.record_submissions = Some(path); + pub fn record_submissions(mut self, path: impl AsRef) -> Self { + debug!("Recording submissions to {:?}", path.as_ref()); + self.record_submissions = Some(path.as_ref().to_path_buf()); self } - pub fn record_events(mut self, path: impl AsRef) -> Self { - let path = match expanduser(path.as_ref()) { - Ok(path) => path, - Err(_) => PathBuf::from(path.as_ref()), - }; - debug!("Recording events to {}", path.display()); - self.record_events = Some(path); + pub fn record_events(mut self, path: impl AsRef) -> Self { + debug!("Recording events to {:?}", path.as_ref()); + self.record_events = Some(path.as_ref().to_path_buf()); self } } diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index 4de42a76de..ec6c652519 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -56,9 +56,9 @@ pub struct Cli { /// Record submissions into file as JSON #[arg(short = 'S', long)] - pub record_submissions: Option, + pub record_submissions: Option, /// Record events into file as JSON #[arg(short = 'E', long)] - pub record_events: Option, + pub record_events: Option, } From 0a13bb010b39443a3d9f40fd77c606c1523bec17 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 14:20:28 -0700 Subject: [PATCH 80/84] ci: build Rust on Windows as part of CI to ensure we guard platform-specific code appropriately --- .github/workflows/rust-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 7e200960d9..eb399cbce3 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -5,6 +5,7 @@ on: - main paths: - "codex-rs/**" + - ".github/**" push: branches: - main @@ -41,6 +42,8 @@ jobs: strategy: fail-fast: false matrix: + # Note: While Codex CLI does not support Windows today, we include + # Windows in CI to ensure the code at least builds there. include: - runner: macos-14 target: aarch64-apple-darwin @@ -50,6 +53,8 @@ jobs: target: x86_64-unknown-linux-musl - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu + - runner: windows-latest + target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v4 From 532da04bb4f994003708c4cab6a65b757856090e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 16:22:32 -0700 Subject: [PATCH 81/84] fix: write logs to ~/.codex/log instead of /tmp --- codex-rs/core/src/config.rs | 31 +++++++++++++++++++++++++++---- codex-rs/tui/src/lib.rs | 26 +++++++++++++++++++------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index c094de5436..b5574ceade 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use dirs::home_dir; use serde::Deserialize; @@ -28,15 +30,36 @@ impl Config { } fn load_from_toml() -> Option { - let mut p = home_dir()?; - p.push(".codex/config.toml"); + let mut p = codex_dir().ok()?; + p.push("config.toml"); let contents = std::fs::read_to_string(&p).ok()?; toml::from_str(&contents).ok() } fn load_instructions() -> Option { - let mut p = home_dir()?; - p.push(".codex/instructions.md"); + let mut p = codex_dir().ok()?; + p.push("instructions.md"); std::fs::read_to_string(&p).ok() } } + +/// Returns the path to the Codex configuration directory, which is `~/.codex`. +/// Does not verify that the directory exists. +pub fn codex_dir() -> std::io::Result { + let mut p = home_dir().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "Could not find home directory", + ) + })?; + p.push(".codex"); + Ok(p) +} + +/// Returns the path to the folder where Codex logs are stored. Does not verify +/// that the directory exists. +pub fn log_dir() -> std::io::Result { + let mut p = codex_dir()?; + p.push("log"); + Ok(p) +} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 4a063de084..d0f5f664a6 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -31,19 +31,31 @@ pub use cli::Cli; pub fn run_main(cli: Cli) -> std::io::Result<()> { assert_env_var_set(); + let log_dir = codex_core::config::log_dir()?; + std::fs::create_dir_all(&log_dir)?; // Open (or create) your log file, appending to it. - let file = OpenOptions::new() - .create(true) - .append(true) - .open("/tmp/codex-rs.log")?; + let mut log_file_opts = OpenOptions::new(); + log_file_opts.create(true).append(true); + + // Ensure the file is only readable and writable by the current user. + // Doing the equivalent to `chmod 600` on Windows is quite a bit more code + // and requires the Windows API crates, so we can reconsider that when + // Codex CLI is officially supported on Windows. + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + log_file_opts.mode(0o600); + } + + let log_file = log_file_opts.open(log_dir.join("codex-tui.log"))?; // Wrap file in non‑blocking writer. - let (non_blocking, _guard) = non_blocking(file); + let (non_blocking, _guard) = non_blocking(log_file); - // use RUST_LOG env var, default to trace for codex crates. + // use RUST_LOG env var, default to info for codex crates. let env_filter = || { EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("codex=trace,codex_tui=trace")) + .unwrap_or_else(|_| EnvFilter::new("codex_core=info,codex_tui=info")) }; // Build layered subscriber: From 0492b91d38724127ccd9a3d6197506c007fcb6a2 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 16:34:29 -0700 Subject: [PATCH 82/84] fix: use os-specific env var to locate .cargo folder --- .github/workflows/rust-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 25394d6a57..56f9225f00 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -38,6 +38,8 @@ jobs: defaults: run: working-directory: codex-rs + env: + CARGO_HOME: ${{ runner.os == 'Windows' && format('{0}\\.cargo', env.USERPROFILE) || format('{0}/.cargo', env.HOME) }} strategy: fail-fast: false @@ -65,10 +67,10 @@ jobs: - uses: actions/cache@v4 with: path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ + ${{ env.CARGO_HOME }}/bin/ + ${{ env.CARGO_HOME }}/registry/index/ + ${{ env.CARGO_HOME }}/registry/cache/ + ${{ env.CARGO_HOME }}/git/db/ ${{ github.workspace }}/codex-rs/target/ key: cargo-${{ matrix.runner }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }} From 707beb21e815120b6a9fa74c91f573370ed10763 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 18:03:11 -0700 Subject: [PATCH 83/84] feat: load defaults into Config and introduce ConfigOverrides --- codex-rs/core/src/approval_mode_cli_arg.rs | 4 +- codex-rs/core/src/codex.rs | 2 - codex-rs/core/src/codex_wrapper.rs | 9 ++-- codex-rs/core/src/config.rs | 51 +++++++++++++++------ codex-rs/core/src/protocol.rs | 5 +- codex-rs/core/tests/live_agent.rs | 8 ++-- codex-rs/core/tests/previous_response_id.rs | 8 ++-- codex-rs/core/tests/stream_no_completed.rs | 8 ++-- codex-rs/exec/src/lib.rs | 15 ++++-- codex-rs/repl/src/cli.rs | 5 +- codex-rs/repl/src/lib.rs | 19 +++++--- codex-rs/tui/src/chatwidget.rs | 9 ++-- codex-rs/tui/src/cli.rs | 5 +- codex-rs/tui/src/lib.rs | 15 ++++-- 14 files changed, 107 insertions(+), 56 deletions(-) diff --git a/codex-rs/core/src/approval_mode_cli_arg.rs b/codex-rs/core/src/approval_mode_cli_arg.rs index eb90b24d87..0da6a89efc 100644 --- a/codex-rs/core/src/approval_mode_cli_arg.rs +++ b/codex-rs/core/src/approval_mode_cli_arg.rs @@ -6,7 +6,7 @@ use clap::ValueEnum; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; -#[derive(Clone, Debug, ValueEnum)] +#[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum ApprovalModeCliArg { /// Run all commands without asking for user approval. @@ -24,7 +24,7 @@ pub enum ApprovalModeCliArg { Never, } -#[derive(Clone, Debug, ValueEnum)] +#[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum SandboxModeCliArg { /// Network syscalls will be blocked diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cfeb7e401f..2f80e505c0 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -36,7 +36,6 @@ use crate::exec::process_exec_tool_call; use crate::exec::ExecParams; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; -use crate::flags::OPENAI_DEFAULT_MODEL; use crate::flags::OPENAI_STREAM_MAX_RETRIES; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; @@ -486,7 +485,6 @@ async fn submission_loop( sandbox_policy, disable_response_storage, } => { - let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); let client = ModelClient::new(model.clone()); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 8d19683ffa..991c134fbb 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -11,7 +11,6 @@ use crate::protocol::Submission; use crate::util::notify_on_sigint; use crate::Codex; use tokio::sync::Notify; -use tracing::debug; /// Spawn a new [`Codex`] and initialise the session. /// @@ -19,19 +18,17 @@ use tracing::debug; /// is received as a response to the initial `ConfigureSession` submission so /// that callers can surface the information to the UI. pub async fn init_codex( + config: Config, approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, disable_response_storage: bool, - model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); - let config = Config::load().unwrap_or_default(); - debug!("loaded config: {config:?}"); let codex = CodexWrapper::new(Codex::spawn(ctrl_c.clone())?); let init_id = codex .submit(Op::ConfigureSession { - model: model_override.or_else(|| config.model.clone()), - instructions: config.instructions, + model: config.model.clone(), + instructions: config.instructions.clone(), approval_policy, sandbox_policy, disable_response_storage, diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index b5574ceade..27cd900f8a 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,32 +1,53 @@ -use std::path::PathBuf; - +use crate::flags::OPENAI_DEFAULT_MODEL; +use crate::protocol::AskForApproval; use dirs::home_dir; use serde::Deserialize; +use std::path::PathBuf; -/// Embedded fallback instructions that mirror the TypeScript CLI’s default system prompt. These -/// are compiled into the binary so a clean install behaves correctly even if the user has not -/// created `~/.codex/instructions.md`. +/// Embedded fallback instructions that mirror the TypeScript CLI’s default +/// system prompt. These are compiled into the binary so a clean install behaves +/// correctly even if the user has not created `~/.codex/instructions.md`. const EMBEDDED_INSTRUCTIONS: &str = include_str!("../prompt.md"); +/// Application configuration loaded from disk and merged with overrides. #[derive(Default, Deserialize, Debug, Clone)] pub struct Config { - pub model: Option, + /// Optional override of model selection. + #[serde(default = "default_model")] + pub model: String, + /// Default approval policy for executing commands. + #[serde(default)] + pub approval_policy: AskForApproval, + /// System instructions. pub instructions: Option, } +/// Optional overrides for user configuration (e.g., from CLI flags). +#[derive(Default, Debug, Clone)] +pub struct ConfigOverrides { + pub model: Option, + pub approval_policy: Option, +} + impl Config { - /// Load ~/.codex/config.toml and ~/.codex/instructions.md (if present). - /// Returns `None` if neither file exists. - pub fn load() -> Option { + /// Load configuration, optionally applying overrides (CLI flags). Merges + /// ~/.codex/config.toml, ~/.codex/instructions.md, embedded defaults, and + /// any values provided in `overrides` (highest precedence). + pub fn load_with_overrides(overrides: ConfigOverrides) -> Self { let mut cfg: Config = Self::load_from_toml().unwrap_or_default(); - // Highest precedence → user‑provided ~/.codex/instructions.md (if present) - // Fallback → embedded default instructions baked into the binary - + // Instructions: user-provided instructions.md > embedded default. cfg.instructions = Self::load_instructions().or_else(|| Some(EMBEDDED_INSTRUCTIONS.to_string())); - Some(cfg) + // Apply overrides. + if let Some(model) = overrides.model { + cfg.model = model; + } + if let Some(policy) = overrides.approval_policy { + cfg.approval_policy = policy; + } + cfg } fn load_from_toml() -> Option { @@ -43,6 +64,10 @@ impl Config { } } +fn default_model() -> String { + OPENAI_DEFAULT_MODEL.to_string() +} + /// Returns the path to the Codex configuration directory, which is `~/.codex`. /// Does not verify that the directory exists. pub fn codex_dir() -> std::io::Result { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 96c4ea4832..8d019d784c 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -26,7 +26,7 @@ pub enum Op { /// Configure the model session. ConfigureSession { /// If not specified, server will use its default model. - model: Option, + model: String, /// Model instructions instructions: Option, /// When to escalate for approval for execution @@ -66,11 +66,12 @@ pub enum Op { } /// Determines how liberally commands are auto‑approved by the system. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AskForApproval { /// Under this policy, only “known safe” commands—as determined by /// `is_safe_command()`—that **only read files** are auto‑approved. /// Everything else will ask the user to approve. + #[default] UnlessAllowListed, /// In addition to everything allowed by **`Suggest`**, commands that diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 823cd73a01..c74d3e17dc 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -17,7 +17,8 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; @@ -47,13 +48,14 @@ async fn spawn_codex() -> Codex { let agent = Codex::spawn(std::sync::Arc::new(Notify::new())).unwrap(); + let config = Config::load_with_overrides(ConfigOverrides::default()); agent .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index de1309e856..bb04d54d1e 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; @@ -87,13 +88,14 @@ async fn keeps_previous_response_id_between_tasks() { let codex = Codex::spawn(std::sync::Arc::new(tokio::sync::Notify::new())).unwrap(); // Init session + let config = Config::load_with_overrides(ConfigOverrides::default()); codex .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index c732a5fdbb..c33bfef753 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -3,7 +3,8 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; @@ -70,13 +71,14 @@ async fn retries_on_early_close() { let codex = Codex::spawn(std::sync::Arc::new(tokio::sync::Notify::new())).unwrap(); + let config = Config::load_with_overrides(ConfigOverrides::default()); codex .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index ab7d735e0f..49ca07b623 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -3,7 +3,8 @@ use std::sync::Arc; pub use cli::Cli; use codex_core::codex_wrapper; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::FileChange; @@ -47,15 +48,19 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { std::process::exit(1); } - // TODO(mbolin): We are reworking the CLI args right now, so this will - // likely come from a new --execution-policy arg. - let approval_policy = AskForApproval::Never; + // Load configuration and determine approval policy + let overrides = ConfigOverrides { + model: model.clone(), + ..Default::default() + }; + let config = Config::load_with_overrides(overrides); + let approval_policy = config.approval_policy; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + config, approval_policy, sandbox_policy, disable_response_storage, - model, ) .await?; let codex = Arc::new(codex_wrapper); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index ec6c652519..af1f1b9c17 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -34,8 +34,9 @@ pub struct Cli { pub no_ansi: bool, /// Configure when the model requires human approval before executing a command. - #[arg(long = "ask-for-approval", short = 'a', value_enum, default_value_t = ApprovalModeCliArg::OnFailure)] - pub approval_policy: ApprovalModeCliArg, + /// Overrides the value in ~/.codex/config.toml if provided. + #[arg(long = "ask-for-approval", short = 'a', value_enum)] + pub approval_policy: Option, /// Configure the process restrictions when a command is executed. /// diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 0f9c47e49b..a027efe702 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::sync::Arc; use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol; use codex_core::protocol::FileChange; use codex_core::util::is_inside_git_repo; @@ -75,12 +76,17 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // Initialize logging before any other work so early errors are captured. init_logger(cli.verbose, !cli.no_ansi); - let config = Config::load().unwrap_or_default(); + // Load config file and apply CLI overrides (model & approval policy) + let overrides = ConfigOverrides { + model: cli.model.clone(), + approval_policy: cli.approval_policy.map(Into::into), + }; + let config = Config::load_with_overrides(overrides); codex_main(cli, config, ctrl_c).await } -async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::Result<()> { +async fn codex_main(cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::Result<()> { let mut builder = Codex::builder(); if let Some(path) = cli.record_submissions { builder = builder.record_submissions(path); @@ -90,12 +96,13 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R } let codex = builder.spawn(Arc::clone(&ctrl_c))?; let init_id = random_id(); + // Determine effective approval policy: CLI flag > config file > default let init = protocol::Submission { id: init_id.clone(), op: protocol::Op::ConfigureSession { - model: cli.model.or(cfg.model), + model: cfg.model, instructions: cfg.instructions, - approval_policy: cli.approval_policy.into(), + approval_policy: cfg.approval_policy, sandbox_policy: cli.sandbox_policy.into(), disable_response_storage: cli.disable_response_storage, }, @@ -133,8 +140,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R // run loop let mut reader = InputReader::new(ctrl_c.clone()); loop { - let text = match cli.prompt.take() { - Some(input) => input, + let text = match &cli.prompt { + Some(input) => input.clone(), None => match reader.request_input().await? { Some(input) => input, None => { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index b852638cc2..46f83fdd8d 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -3,6 +3,7 @@ use std::sync::mpsc::Sender; use std::sync::Arc; use codex_core::codex_wrapper::init_codex; +use codex_core::config::{Config, ConfigOverrides}; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; @@ -64,18 +65,20 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - // Initialize session; storage enabled by default + // Load config and initialize session + let overrides = ConfigOverrides { model: model.clone(), ..Default::default() }; + let config = Config::load_with_overrides(overrides); let (codex, session_event, _ctrl_c) = match init_codex( + config, approval_policy, sandbox_policy, disable_response_storage, - model, ) .await { Ok(vals) => vals, Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. + // TODO: surface this error to the user. tracing::error!("failed to initialize codex: {e}"); return; } diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index db25ad2b3c..bdbb9fe7dc 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -18,8 +18,9 @@ pub struct Cli { pub model: Option, /// Configure when the model requires human approval before executing a command. - #[arg(long = "ask-for-approval", short = 'a', value_enum, default_value_t = ApprovalModeCliArg::OnFailure)] - pub approval_policy: ApprovalModeCliArg, + /// Overrides the value in ~/.codex/config.toml if provided. + #[arg(long = "ask-for-approval", short = 'a', value_enum)] + pub approval_policy: Option, /// Configure the process restrictions when a command is executed. /// diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index d0f5f664a6..c9f903ba15 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -4,6 +4,8 @@ #![deny(clippy::print_stdout, clippy::print_stderr)] use app::App; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::util::is_inside_git_repo; use log_layer::TuiLogLayer; use std::fs::OpenOptions; @@ -113,21 +115,26 @@ fn run_ratatui_app( let mut terminal = tui::init()?; terminal.clear()?; + // Load configuration and destructure CLI flags let Cli { prompt, images, - approval_policy, + approval_policy: cli_approval, sandbox_policy: sandbox, model, disable_response_storage, .. } = cli; + // Apply CLI overrides and load merged configuration + let overrides = ConfigOverrides { + model: model.clone(), + approval_policy: cli_approval.map(Into::into), + }; + let cfg = Config::load_with_overrides(overrides); - let approval_policy = approval_policy.into(); let sandbox_policy = sandbox.into(); - let mut app = App::new( - approval_policy, + cfg.approval_policy, sandbox_policy, prompt, show_git_warning, From 86c2498972f896fb50b2809b30880d63ceb27b6a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 25 Apr 2025 18:03:11 -0700 Subject: [PATCH 84/84] feat: load defaults into Config and introduce ConfigOverrides --- codex-rs/core/src/approval_mode_cli_arg.rs | 4 +- codex-rs/core/src/codex.rs | 2 - codex-rs/core/src/codex_wrapper.rs | 9 ++-- codex-rs/core/src/config.rs | 51 +++++++++++++++------ codex-rs/core/src/protocol.rs | 5 +- codex-rs/core/tests/live_agent.rs | 8 ++-- codex-rs/core/tests/previous_response_id.rs | 8 ++-- codex-rs/core/tests/stream_no_completed.rs | 8 ++-- codex-rs/exec/src/lib.rs | 15 ++++-- codex-rs/repl/src/cli.rs | 5 +- codex-rs/repl/src/lib.rs | 19 +++++--- codex-rs/tui/src/chatwidget.rs | 13 ++++-- codex-rs/tui/src/cli.rs | 5 +- codex-rs/tui/src/lib.rs | 15 ++++-- 14 files changed, 111 insertions(+), 56 deletions(-) diff --git a/codex-rs/core/src/approval_mode_cli_arg.rs b/codex-rs/core/src/approval_mode_cli_arg.rs index eb90b24d87..0da6a89efc 100644 --- a/codex-rs/core/src/approval_mode_cli_arg.rs +++ b/codex-rs/core/src/approval_mode_cli_arg.rs @@ -6,7 +6,7 @@ use clap::ValueEnum; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; -#[derive(Clone, Debug, ValueEnum)] +#[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum ApprovalModeCliArg { /// Run all commands without asking for user approval. @@ -24,7 +24,7 @@ pub enum ApprovalModeCliArg { Never, } -#[derive(Clone, Debug, ValueEnum)] +#[derive(Clone, Copy, Debug, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum SandboxModeCliArg { /// Network syscalls will be blocked diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cfeb7e401f..2f80e505c0 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -36,7 +36,6 @@ use crate::exec::process_exec_tool_call; use crate::exec::ExecParams; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; -use crate::flags::OPENAI_DEFAULT_MODEL; use crate::flags::OPENAI_STREAM_MAX_RETRIES; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; @@ -486,7 +485,6 @@ async fn submission_loop( sandbox_policy, disable_response_storage, } => { - let model = model.unwrap_or_else(|| OPENAI_DEFAULT_MODEL.to_string()); info!(model, "Configuring session"); let client = ModelClient::new(model.clone()); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 8d19683ffa..991c134fbb 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -11,7 +11,6 @@ use crate::protocol::Submission; use crate::util::notify_on_sigint; use crate::Codex; use tokio::sync::Notify; -use tracing::debug; /// Spawn a new [`Codex`] and initialise the session. /// @@ -19,19 +18,17 @@ use tracing::debug; /// is received as a response to the initial `ConfigureSession` submission so /// that callers can surface the information to the UI. pub async fn init_codex( + config: Config, approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, disable_response_storage: bool, - model_override: Option, ) -> anyhow::Result<(CodexWrapper, Event, Arc)> { let ctrl_c = notify_on_sigint(); - let config = Config::load().unwrap_or_default(); - debug!("loaded config: {config:?}"); let codex = CodexWrapper::new(Codex::spawn(ctrl_c.clone())?); let init_id = codex .submit(Op::ConfigureSession { - model: model_override.or_else(|| config.model.clone()), - instructions: config.instructions, + model: config.model.clone(), + instructions: config.instructions.clone(), approval_policy, sandbox_policy, disable_response_storage, diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index b5574ceade..27cd900f8a 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,32 +1,53 @@ -use std::path::PathBuf; - +use crate::flags::OPENAI_DEFAULT_MODEL; +use crate::protocol::AskForApproval; use dirs::home_dir; use serde::Deserialize; +use std::path::PathBuf; -/// Embedded fallback instructions that mirror the TypeScript CLI’s default system prompt. These -/// are compiled into the binary so a clean install behaves correctly even if the user has not -/// created `~/.codex/instructions.md`. +/// Embedded fallback instructions that mirror the TypeScript CLI’s default +/// system prompt. These are compiled into the binary so a clean install behaves +/// correctly even if the user has not created `~/.codex/instructions.md`. const EMBEDDED_INSTRUCTIONS: &str = include_str!("../prompt.md"); +/// Application configuration loaded from disk and merged with overrides. #[derive(Default, Deserialize, Debug, Clone)] pub struct Config { - pub model: Option, + /// Optional override of model selection. + #[serde(default = "default_model")] + pub model: String, + /// Default approval policy for executing commands. + #[serde(default)] + pub approval_policy: AskForApproval, + /// System instructions. pub instructions: Option, } +/// Optional overrides for user configuration (e.g., from CLI flags). +#[derive(Default, Debug, Clone)] +pub struct ConfigOverrides { + pub model: Option, + pub approval_policy: Option, +} + impl Config { - /// Load ~/.codex/config.toml and ~/.codex/instructions.md (if present). - /// Returns `None` if neither file exists. - pub fn load() -> Option { + /// Load configuration, optionally applying overrides (CLI flags). Merges + /// ~/.codex/config.toml, ~/.codex/instructions.md, embedded defaults, and + /// any values provided in `overrides` (highest precedence). + pub fn load_with_overrides(overrides: ConfigOverrides) -> Self { let mut cfg: Config = Self::load_from_toml().unwrap_or_default(); - // Highest precedence → user‑provided ~/.codex/instructions.md (if present) - // Fallback → embedded default instructions baked into the binary - + // Instructions: user-provided instructions.md > embedded default. cfg.instructions = Self::load_instructions().or_else(|| Some(EMBEDDED_INSTRUCTIONS.to_string())); - Some(cfg) + // Apply overrides. + if let Some(model) = overrides.model { + cfg.model = model; + } + if let Some(policy) = overrides.approval_policy { + cfg.approval_policy = policy; + } + cfg } fn load_from_toml() -> Option { @@ -43,6 +64,10 @@ impl Config { } } +fn default_model() -> String { + OPENAI_DEFAULT_MODEL.to_string() +} + /// Returns the path to the Codex configuration directory, which is `~/.codex`. /// Does not verify that the directory exists. pub fn codex_dir() -> std::io::Result { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 96c4ea4832..8d019d784c 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -26,7 +26,7 @@ pub enum Op { /// Configure the model session. ConfigureSession { /// If not specified, server will use its default model. - model: Option, + model: String, /// Model instructions instructions: Option, /// When to escalate for approval for execution @@ -66,11 +66,12 @@ pub enum Op { } /// Determines how liberally commands are auto‑approved by the system. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AskForApproval { /// Under this policy, only “known safe” commands—as determined by /// `is_safe_command()`—that **only read files** are auto‑approved. /// Everything else will ask the user to approve. + #[default] UnlessAllowListed, /// In addition to everything allowed by **`Suggest`**, commands that diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 823cd73a01..c74d3e17dc 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -17,7 +17,8 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; @@ -47,13 +48,14 @@ async fn spawn_codex() -> Codex { let agent = Codex::spawn(std::sync::Arc::new(Notify::new())).unwrap(); + let config = Config::load_with_overrides(ConfigOverrides::default()); agent .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/core/tests/previous_response_id.rs b/codex-rs/core/tests/previous_response_id.rs index de1309e856..bb04d54d1e 100644 --- a/codex-rs/core/tests/previous_response_id.rs +++ b/codex-rs/core/tests/previous_response_id.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; @@ -87,13 +88,14 @@ async fn keeps_previous_response_id_between_tasks() { let codex = Codex::spawn(std::sync::Arc::new(tokio::sync::Notify::new())).unwrap(); // Init session + let config = Config::load_with_overrides(ConfigOverrides::default()); codex .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index c732a5fdbb..c33bfef753 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -3,7 +3,8 @@ use std::time::Duration; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; @@ -70,13 +71,14 @@ async fn retries_on_early_close() { let codex = Codex::spawn(std::sync::Arc::new(tokio::sync::Notify::new())).unwrap(); + let config = Config::load_with_overrides(ConfigOverrides::default()); codex .submit(Submission { id: "init".into(), op: Op::ConfigureSession { - model: None, + model: config.model, instructions: None, - approval_policy: AskForApproval::OnFailure, + approval_policy: config.approval_policy, sandbox_policy: SandboxPolicy::NetworkAndFileWriteRestricted, disable_response_storage: false, }, diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index ab7d735e0f..49ca07b623 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -3,7 +3,8 @@ use std::sync::Arc; pub use cli::Cli; use codex_core::codex_wrapper; -use codex_core::protocol::AskForApproval; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::FileChange; @@ -47,15 +48,19 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { std::process::exit(1); } - // TODO(mbolin): We are reworking the CLI args right now, so this will - // likely come from a new --execution-policy arg. - let approval_policy = AskForApproval::Never; + // Load configuration and determine approval policy + let overrides = ConfigOverrides { + model: model.clone(), + ..Default::default() + }; + let config = Config::load_with_overrides(overrides); + let approval_policy = config.approval_policy; let sandbox_policy = SandboxPolicy::NetworkAndFileWriteRestricted; let (codex_wrapper, event, ctrl_c) = codex_wrapper::init_codex( + config, approval_policy, sandbox_policy, disable_response_storage, - model, ) .await?; let codex = Arc::new(codex_wrapper); diff --git a/codex-rs/repl/src/cli.rs b/codex-rs/repl/src/cli.rs index ec6c652519..af1f1b9c17 100644 --- a/codex-rs/repl/src/cli.rs +++ b/codex-rs/repl/src/cli.rs @@ -34,8 +34,9 @@ pub struct Cli { pub no_ansi: bool, /// Configure when the model requires human approval before executing a command. - #[arg(long = "ask-for-approval", short = 'a', value_enum, default_value_t = ApprovalModeCliArg::OnFailure)] - pub approval_policy: ApprovalModeCliArg, + /// Overrides the value in ~/.codex/config.toml if provided. + #[arg(long = "ask-for-approval", short = 'a', value_enum)] + pub approval_policy: Option, /// Configure the process restrictions when a command is executed. /// diff --git a/codex-rs/repl/src/lib.rs b/codex-rs/repl/src/lib.rs index 0f9c47e49b..a027efe702 100644 --- a/codex-rs/repl/src/lib.rs +++ b/codex-rs/repl/src/lib.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::sync::Arc; use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol; use codex_core::protocol::FileChange; use codex_core::util::is_inside_git_repo; @@ -75,12 +76,17 @@ pub async fn run_main(cli: Cli) -> anyhow::Result<()> { // Initialize logging before any other work so early errors are captured. init_logger(cli.verbose, !cli.no_ansi); - let config = Config::load().unwrap_or_default(); + // Load config file and apply CLI overrides (model & approval policy) + let overrides = ConfigOverrides { + model: cli.model.clone(), + approval_policy: cli.approval_policy.map(Into::into), + }; + let config = Config::load_with_overrides(overrides); codex_main(cli, config, ctrl_c).await } -async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::Result<()> { +async fn codex_main(cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::Result<()> { let mut builder = Codex::builder(); if let Some(path) = cli.record_submissions { builder = builder.record_submissions(path); @@ -90,12 +96,13 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R } let codex = builder.spawn(Arc::clone(&ctrl_c))?; let init_id = random_id(); + // Determine effective approval policy: CLI flag > config file > default let init = protocol::Submission { id: init_id.clone(), op: protocol::Op::ConfigureSession { - model: cli.model.or(cfg.model), + model: cfg.model, instructions: cfg.instructions, - approval_policy: cli.approval_policy.into(), + approval_policy: cfg.approval_policy, sandbox_policy: cli.sandbox_policy.into(), disable_response_storage: cli.disable_response_storage, }, @@ -133,8 +140,8 @@ async fn codex_main(mut cli: Cli, cfg: Config, ctrl_c: Arc) -> anyhow::R // run loop let mut reader = InputReader::new(ctrl_c.clone()); loop { - let text = match cli.prompt.take() { - Some(input) => input, + let text = match &cli.prompt { + Some(input) => input.clone(), None => match reader.request_input().await? { Some(input) => input, None => { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index b852638cc2..94822c5f09 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -3,6 +3,8 @@ use std::sync::mpsc::Sender; use std::sync::Arc; use codex_core::codex_wrapper::init_codex; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::protocol::AskForApproval; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; @@ -64,18 +66,23 @@ impl ChatWidget<'_> { let app_event_tx_clone = app_event_tx.clone(); // Create the Codex asynchronously so the UI loads as quickly as possible. tokio::spawn(async move { - // Initialize session; storage enabled by default + // Load config and initialize session + let overrides = ConfigOverrides { + model: model.clone(), + ..Default::default() + }; + let config = Config::load_with_overrides(overrides); let (codex, session_event, _ctrl_c) = match init_codex( + config, approval_policy, sandbox_policy, disable_response_storage, - model, ) .await { Ok(vals) => vals, Err(e) => { - // TODO(mbolin): This error needs to be surfaced to the user. + // TODO: surface this error to the user. tracing::error!("failed to initialize codex: {e}"); return; } diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index db25ad2b3c..bdbb9fe7dc 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -18,8 +18,9 @@ pub struct Cli { pub model: Option, /// Configure when the model requires human approval before executing a command. - #[arg(long = "ask-for-approval", short = 'a', value_enum, default_value_t = ApprovalModeCliArg::OnFailure)] - pub approval_policy: ApprovalModeCliArg, + /// Overrides the value in ~/.codex/config.toml if provided. + #[arg(long = "ask-for-approval", short = 'a', value_enum)] + pub approval_policy: Option, /// Configure the process restrictions when a command is executed. /// diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index d0f5f664a6..c9f903ba15 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -4,6 +4,8 @@ #![deny(clippy::print_stdout, clippy::print_stderr)] use app::App; +use codex_core::config::Config; +use codex_core::config::ConfigOverrides; use codex_core::util::is_inside_git_repo; use log_layer::TuiLogLayer; use std::fs::OpenOptions; @@ -113,21 +115,26 @@ fn run_ratatui_app( let mut terminal = tui::init()?; terminal.clear()?; + // Load configuration and destructure CLI flags let Cli { prompt, images, - approval_policy, + approval_policy: cli_approval, sandbox_policy: sandbox, model, disable_response_storage, .. } = cli; + // Apply CLI overrides and load merged configuration + let overrides = ConfigOverrides { + model: model.clone(), + approval_policy: cli_approval.map(Into::into), + }; + let cfg = Config::load_with_overrides(overrides); - let approval_policy = approval_policy.into(); let sandbox_policy = sandbox.into(); - let mut app = App::new( - approval_policy, + cfg.approval_policy, sandbox_policy, prompt, show_git_warning,