From 6f1d48b4894d828b5735fe5c4baca96766496567 Mon Sep 17 00:00:00 2001 From: "Rai (Michael Pokorny)" Date: Wed, 25 Jun 2025 03:06:44 -0700 Subject: [PATCH] wip --- .pre-commit-config.yaml | 19 +--- agentydragon/README.md | 4 + agentydragon/prompts/manager.md | 25 ++++- agentydragon/tools/check_task_cycles.py | 62 ----------- agentydragon/tools/check_task_frontmatter.py | 49 --------- agentydragon/tools/check_tasks.py | 108 +++++++++++++++++++ agentydragon/tools/check_tasks_files.py | 26 ----- codex-rs/README.md | 56 ++++++++++ codex-rs/config.md | 8 ++ codex-rs/core/README.md | 17 +++ codex-rs/core/src/client_common.rs | 30 +++++- 11 files changed, 247 insertions(+), 157 deletions(-) delete mode 100755 agentydragon/tools/check_task_cycles.py delete mode 100755 agentydragon/tools/check_task_frontmatter.py create mode 100644 agentydragon/tools/check_tasks.py delete mode 100755 agentydragon/tools/check_tasks_files.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54d307470e..6d4709cf12 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,12 @@ repos: - repo: local hooks: - - id: check-task-frontmatter - name: Check agentydragon task frontmatter - entry: python3 agentydragon/tools/check_task_frontmatter.py + - id: check-tasks + name: Run all task-directory validation checks + entry: python3 agentydragon/tools/check_tasks.py language: python additional_dependencies: [PyYAML, toml, pydantic] - files: ^agentydragon/tasks/(?:\.done/)?[0-9]{2}-.*\.md$ - - id: check-task-dependency-cycles - name: Check for circular task dependencies - entry: python3 agentydragon/tools/check_task_cycles.py - language: python - additional_dependencies: [toml, pydantic] - files: ^agentydragon/tasks/(?:\.done/)?[0-9]{2}-.*\.md$ - - id: enforce-tasks-md-only - name: Enforce only Markdown files in agentydragon/tasks/ - entry: python3 agentydragon/tools/check_tasks_files.py - language: python - files: ^agentydragon/tasks/(?!\.worktrees/)(?!\.done/).* + files: ^agentydragon/tasks/.* - id: cargo-build name: Check Rust workspace builds entry: bash -lc 'cd codex-rs && RUSTFLAGS="-D warnings" cargo build --workspace --locked' diff --git a/agentydragon/README.md b/agentydragon/README.md index f5d50fb0df..e2015f6b55 100644 --- a/agentydragon/README.md +++ b/agentydragon/README.md @@ -43,6 +43,10 @@ This file documents the changes introduced on the `agentydragon` branch - Updated Markdown rendering in chat UI and logs to honor compact mode globally (diffs, docs, help messages). - Added unit tests covering H1–H6 heading spacing for both compact and default modes. +## codex-rs: document MCP servers example in README +- Added an inline TOML snippet under “Model Context Protocol Support” in `codex-rs/README.md` showing how to configure external `mcp_servers` entries in `~/.codex/config.toml`. +- Documented `codex mcp` behavior: JSON-RPC over stdin/stdout, optional sandbox, no ephemeral container, default `codex` tool schema, and example ListTools/CallTool schema. + ## Documentation tasks Tasks live under `agentydragon/tasks/` as individual Markdown files. Please update each task’s **Status** and **Implementation** sections in place rather than maintaining a static list here. diff --git a/agentydragon/prompts/manager.md b/agentydragon/prompts/manager.md index 749cc8ea0b..8827c0d54c 100644 --- a/agentydragon/prompts/manager.md +++ b/agentydragon/prompts/manager.md @@ -5,7 +5,7 @@ Refer to `agentydragon/WORKFLOW.md` for the standard Developer→Commit→Orches Your responsibilities include: - **Reading documentation**: Load and understand all relevant docs in this repo (especially those defining task, worktree, and branch conventions, as well as each task file and top‑level README files). -- **Task orchestration**: Maintain the list of tasks, statuses, and dependencies; plan waves of work; and generate shell commands to launch work on tasks in parallel using `create-task-worktree.sh` with `--agent` and `--tmux`. +- **Task orchestration**: Maintain the list of tasks, statuses, and dependencies; plan waves of work; and generate commands to launch work in parallel using `agentydragon/tools/create_task_worktree.py` (or the legacy `agentydragon/tools/create-task-worktree.sh`) with `--agent` and `--tmux`. - **Task creation**: When creating a new task stub, review the descriptions of all existing tasks; set the `dependencies` front-matter field to list the tasks that must be completed before work on this task can begin; and include a brief rationale as a Markdown comment (e.g., ``) explaining why these dependencies are required and why other tasks are not. - **Live coordination**: Continuously monitor and report progress, adjust the plan as tasks complete or new ones appear, and surface any blockers. @@ -18,10 +18,31 @@ Your responsibilities include: ### First Actions -1. For each task branch (named `agentydragon--`), **without changing the current working directory’s Git HEAD or modifying its status**, create or open a dedicated worktree for that branch (e.g. via `create-task-worktree.sh `) and read the task’s Markdown copy under that worktree’s `agentydragon/tasks/` to extract and list the task number, title, live **Status**, and dependencies. *(Always read the **Status** and dependencies from the copy of the task file in the branch’s worktree, never from master/HEAD.)* +1. For each task branch (named `agentydragon--`), **without changing the current working directory’s Git HEAD or modifying its status**, create or open a dedicated worktree for that branch (e.g. via `agentydragon/tools/create_task_worktree.py `) and read the task’s Markdown copy in that worktree to extract and list the task number, title, live **Status**, and dependencies. *(Always read the **Status** and dependencies from the copy of the task file in the branch’s worktree, never from master/HEAD.)* 2. Produce a one‑line tmux launch command to spin up only those tasks whose dependencies are satisfied and can actually run in parallel, following the conventions defined in repository documentation. 3. Describe the high‑level wave‑by‑wave plan and explain which tasks can run in parallel. +### Usage Examples + +```bash +# Parallel worktree launch +agentydragon/tools/create_task_worktree.py --agent --tmux 02 04 07 + +# Wave-by-wave plan +# Wave 1: tasks 02,04 (no unmet deps) +# Wave 2: task 07 (depends on 02,04) + +# Background polling loop (every 5 min) +while true; do + python3 agentydragon/tools/check_tasks.py && \ + python3 agentydragon/tools/launch_commit_agent.py $(python3 agentydragon/tools/find_done_tasks.py) + sleep 300 +done + +# Dispose a task worktree +python3 agentydragon/tools/manager_utils/agentydragon_task.py dispose 07 +``` + More functionality and refinements will be added later. Begin by executing these steps and await further instructions. *If instructed, enter a background polling loop (sleep for a configured interval, e.g. 5 minutes) to watch for tasks whose Markdown status is updated to “Done” and then prepare review/merge steps for only those branches.* diff --git a/agentydragon/tools/check_task_cycles.py b/agentydragon/tools/check_task_cycles.py deleted file mode 100755 index aab257f1ad..0000000000 --- a/agentydragon/tools/check_task_cycles.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -""" -check_task_cycles.py: Pre-commit hook to detect circular dependencies among non-merged tasks. -""" -import re -import sys -from pathlib import Path - -# allow importing from manager_utils by adding tool directory to PYTHONPATH -sys.path.insert(0, str(Path(__file__).parent.parent)) -from manager_utils.tasklib import task_dir, load_task - -def main(): - # Load all tasks and separate merged vs non-merged - merged = set() - deps_map = {} - id_to_path = {} - # skip template/plan files and any worktree copies - wt_root = task_dir() / '.worktrees' - for md in task_dir().rglob('[0-9][0-9]-*.md'): - if md.name == 'task-template.md' or md.name.endswith('-plan.md') or md.is_relative_to(wt_root): - continue - meta, _ = load_task(md) - id_to_path[meta.id] = md - if meta.status == 'Merged': - merged.add(meta.id) - else: - # extract numeric dependencies - deps = [d for d in re.findall(r"\d+", meta.dependencies)] - deps_map[meta.id] = deps - - # filter out dependencies on merged tasks - for tid in deps_map: - deps_map[tid] = [d for d in deps_map[tid] if d not in merged] - - # detect cycles via DFS - visited = set() - stack = [] - - def visit(n): - if n in stack: - cycle = stack[stack.index(n):] + [n] - cycle_str = ' -> '.join(cycle) - print(f"Circular dependency detected: {cycle_str}", file=sys.stderr) - print("Paths involved in cycle:", file=sys.stderr) - for tid in cycle: - print(f" {id_to_path.get(tid, tid)}", file=sys.stderr) - sys.exit(1) - if n in visited: - return - stack.append(n) - for m in deps_map.get(n, []): - visit(m) - stack.pop() - visited.add(n) - - for node in deps_map: - if node not in visited: - visit(node) - -if __name__ == '__main__': - main() diff --git a/agentydragon/tools/check_task_frontmatter.py b/agentydragon/tools/check_task_frontmatter.py deleted file mode 100755 index a6c98328e0..0000000000 --- a/agentydragon/tools/check_task_frontmatter.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -""" -check_task_frontmatter.py: Validate structured YAML frontmatter in task Markdown files. - -This script ensures each task file under agentydragon/tasks/ has a YAML frontmatter block -with the required keys: id, title, status, summary, and goal. -It also enforces that `status` is one of the allowed enum values. - -Usage: - python3 agentydragon/tools/check_task_frontmatter.py - -Returns exit code 0 if all files pass, 1 otherwise. -""" - -import sys - -from manager_utils import tasklib - -try: - import yaml -except ImportError: - print("Missing dependency: PyYAML is required to run this script.", file=sys.stderr) - sys.exit(1) - -REQUIRED_KEYS = ["id", "title", "status", "summary", "goal"] -ALLOWED_STATUSES = ["Not started", "Started", "Needs manual review", "Done", "Cancelled", "Merged", "Reopened"] - -def main(): - failures = 0 - - # skip template/plan files and any worktree copies - wt_root = tasklib.worktree_dir() - for md in tasklib.task_dir().rglob('[0-9][0-9]-*.md'): - if md.name == 'task-template.md' or md.name.endswith('-plan.md') or md.is_relative_to(wt_root): - continue - try: - task, _body = tasklib.load_task(md) - except ValueError as e: - print(f"{md}: {e}", file=sys.stderr) - failures += 1 - - if failures: - print(f"\nFound {failures} frontmatter errors.", file=sys.stderr) - sys.exit(1) - print("All task frontmatter OK.") - sys.exit(0) - -if __name__ == '__main__': - main() diff --git a/agentydragon/tools/check_tasks.py b/agentydragon/tools/check_tasks.py new file mode 100644 index 0000000000..9be4545610 --- /dev/null +++ b/agentydragon/tools/check_tasks.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +check_tasks.py: Run all task-directory validation checks in one go. + - Ensure task Markdown frontmatter parses and validates (id, title, status, etc.). + - Detect circular dependencies among non-merged tasks. + - Enforce only .md files under agentydragon/tasks/ (excluding .worktrees/ and .done/). +""" +import re +import sys +from pathlib import Path + +import toml +import yaml +from manager_utils.tasklib import TaskMeta, task_dir, worktree_dir, load_task + +def check_file_types(): + failures = [] + wt_root = worktree_dir() + done_root = task_dir() / '.done' + for f in task_dir().iterdir(): + if not f.is_file(): + continue + if f.is_relative_to(wt_root) or f.is_relative_to(done_root): + continue + if f.suffix.lower() != '.md': + failures.append(f) + return failures + +def check_frontmatter(): + failures = [] + wt_root = worktree_dir() + for md in task_dir().rglob('[0-9][0-9]-*.md'): + if md.name in ('task-template.md',) or md.name.endswith('-plan.md') or md.is_relative_to(wt_root): + continue + try: + load_task(md) + except Exception as e: + failures.append((md, str(e))) + return failures + +def check_cycles(): + merged = set() + deps_map = {} + wt_root = worktree_dir() + for md in task_dir().rglob('[0-9][0-9]-*.md'): + if md.name in ('task-template.md',) or md.name.endswith('-plan.md') or md.is_relative_to(wt_root): + continue + meta, _ = load_task(md) + if meta.status == 'Merged': + merged.add(meta.id) + else: + deps = re.findall(r"\d+", meta.dependencies) + deps_map[meta.id] = [d for d in deps if d not in merged] + + failures = [] + visited = set() + stack = [] + + def visit(n): + if n in stack: + cycle = stack[stack.index(n):] + [n] + failures.append(cycle) + return + if n in visited: + return + stack.append(n) + for m in deps_map.get(n, []): + visit(m) + stack.pop() + visited.add(n) + + for node in deps_map: + visit(node) + return failures + +def main(): + err = False + + # File type check + ft_fail = check_file_types() + if ft_fail: + print('Non-md files under tasks/:', file=sys.stderr) + for f in ft_fail: + print(f' {f}', file=sys.stderr) + err = True + + # Frontmatter check + fm_fail = check_frontmatter() + if fm_fail: + print('\nFrontmatter errors:', file=sys.stderr) + for md, msg in fm_fail: + print(f' {md}: {msg}', file=sys.stderr) + err = True + + # Dependency cycles + cyc_fail = check_cycles() + if cyc_fail: + print('\nCircular dependency errors:', file=sys.stderr) + for cycle in cyc_fail: + print(' ' + ' -> '.join(cycle), file=sys.stderr) + err = True + + if err: + sys.exit(1) + print('All task checks passed.') + +if __name__ == '__main__': + main() diff --git a/agentydragon/tools/check_tasks_files.py b/agentydragon/tools/check_tasks_files.py deleted file mode 100755 index 781320a4fe..0000000000 --- a/agentydragon/tools/check_tasks_files.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -""" -check_tasks_files.py: Pre-commit hook to ensure only Markdown files in agentydragon/tasks/ (excluding .worktrees and .done). -""" -import sys -from pathlib import Path - -def main(): - bad = [] - for f in sys.argv[1:]: - p = Path(f) - # skip worktree copies and done archives - if p.is_relative_to(Path('agentydragon/tasks/.worktrees')) or p.is_relative_to(Path('agentydragon/tasks/.done')): - continue - # allow only .md files - if p.suffix.lower() != '.md': - bad.append(f) - if bad: - print('Error: only Markdown (.md) files are allowed under agentydragon/tasks/:', file=sys.stderr) - for f in bad: - print(f' {f}', file=sys.stderr) - sys.exit(1) - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/codex-rs/README.md b/codex-rs/README.md index 3fa0c7dfd6..317a4704e1 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -37,12 +37,68 @@ Codex supports a rich set of configuration options. Note that the Rust CLI uses Codex CLI functions as an MCP client that can connect to MCP servers on startup. See the [`mcp_servers`](./config.md#mcp_servers) section in the configuration documentation for details. +For example, to configure an external MCP server in your `~/.codex/config.toml`: + +```toml +[mcp_servers.server-name] +command = "npx" +args = ["-y", "mcp-server"] +env = { "API_KEY" = "value" } +``` + It is still experimental, but you can also launch Codex as an MCP _server_ by running `codex mcp`. Use the [`@modelcontextprotocol/inspector`](https://github.com/modelcontextprotocol/inspector) to try it out: ```shell npx @modelcontextprotocol/inspector codex mcp ``` +Under the hood, `codex mcp` launches a local MCP server process that communicates over stdin/stdout +using the Model Context Protocol (JSON-RPC messages). It reads `JSONRPCMessage` requests (e.g. +`Initialize`, `ListTools`, `CallTool`) from stdin, handles them, and writes JSONRPCMessage +responses and notifications to stdout. No separate container or VM is spun up and torn down; on +Linux the process is optionally sandboxed via Landlock/seccomp (and on macOS via Seatbelt). +See `codex-rs/mcp-server/src/lib.rs` for the implementation. + +By default, the server advertises a single MCP tool named `codex`. A `ListTools` request +will return this tool along with its input schema (fields: `prompt`, `model`, `profile`, +`cwd`, `approval_policy`, `sandbox_permissions`, `config`). + +#### Example: ListTools and CallTool messages + +```jsonc +// ListTools request +{ "jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {} } + +// ListTools response (abbreviated) +{ "jsonrpc": "2.0", "id": 1, + "result": { "tools": [ + { + "name": "codex", + "description": "Run a Codex session. Accepts configuration parameters matching the Codex Config struct.", + "input_schema": { /* JSON Schema with properties: prompt, model, profile, cwd, approval_policy, sandbox_permissions, config */ } + } + ] } +} + +// CallTool request to run the 'codex' tool with a prompt +{ "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": { "name": "codex", + "arguments": { "prompt": "Hello, world!" } + } +} + +// CallTool response (abbreviated) +{ "jsonrpc": "2.0", "id": 2, + "result": { + "content": [ { "type": "text", "text": "Hello, world! How can I help?", "annotations": null } ], + "is_error": false + } +} +``` + +Calls to the `codex` tool are handled via JSON-RPC `CallTool` requests by spawning an interactive Codex +session based on the provided parameters and streaming the generated text in the `content` field. + ### Notifications You can enable notifications by configuring a script that is run whenever the agent finishes a turn. The [notify documentation](./config.md#notify) includes a detailed example that explains how to get desktop notifications via [terminal-notifier](https://github.com/julienXX/terminal-notifier) on macOS. diff --git a/codex-rs/config.md b/codex-rs/config.md index 301bc2ef8a..a6956d208d 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -396,6 +396,14 @@ hide_agent_reasoning = true # defaults to false Maximum number of bytes to read from an `AGENTS.md` file to include in the instructions sent with the first turn of a session. Defaults to 32 KiB. +## base_instructions_override + +The built-in system prompt (from `prompt.md`) can be overridden or disabled via environment variables: + +- `CODEX_BASE_INSTRUCTIONS_FILE`: If unset, the built-in prompt (`prompt.md`) is used. + If set to a valid file path, that file's contents will be used instead. + If set to an empty string or `-`, no system prompt will be sent. + ## tui Options that are specific to the TUI. diff --git a/codex-rs/core/README.md b/codex-rs/core/README.md index 9b3e59c8af..8b0b1fdc94 100644 --- a/codex-rs/core/README.md +++ b/codex-rs/core/README.md @@ -2,6 +2,23 @@ This crate implements the business logic for Codex. It is designed to be used by the various Codex UIs written in Rust. +## System Prompt Composition + +Codex composes the initial system message that seeds every chat completion turn as follows: + +1. Load the built-in system prompt from `prompt.md` (unless disabled). +2. If the `CODEX_BASE_INSTRUCTIONS_FILE` env var is set, use that file instead of `prompt.md`. +3. Append any user instructions (e.g. from `instructions.md` and merged `AGENTS.md`). +4. Append the apply-patch tool instructions when using GPT-4.1 models. +5. Finally, the user's command or prompt is sent as the first user message. + +The base instructions behavior can be customized via environment variables: + +- `CODEX_BASE_INSTRUCTIONS_FILE`: path to a Markdown file to override the built-in prompt +- `CODEX_DISABLE_BASE_INSTRUCTIONS=1`: skip sending any system prompt entirely + +For implementation details, see `client_common.rs` and `project_doc.rs`. + Though for non-Rust UIs, we are also working to define a _protocol_ for talking to Codex. See: - [Specification](../docs/protocol_v1.md) diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index a2633475df..0ae8d69778 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -36,13 +36,37 @@ pub struct Prompt { } impl Prompt { + /// Assemble the system prompt sent to the model, in order: + /// 1. Base instructions (built-in prompt.md), unless disabled via + /// the CODEX_DISABLE_BASE_INSTRUCTIONS env var. + /// 2. Or, if CODEX_BASE_INSTRUCTIONS_FILE is set, load that file instead of the built-in prompt. + /// 3. User instructions (e.g. from instructions.md and AGENTS.md), if any. + /// 4. Apply-patch tool instructions when using GPT-4.1 models. pub(crate) fn get_full_instructions(&self, model: &str) -> Cow { - let mut sections: Vec<&str> = vec![BASE_INSTRUCTIONS]; + // Determine base instructions or override/disable via CODEX_BASE_INSTRUCTIONS_FILE + let mut sections = Vec::new(); + match std::env::var("CODEX_BASE_INSTRUCTIONS_FILE") { + Ok(ref path) if !path.is_empty() && path != "-" => { + // Override built-in prompt: read file or abort + let contents = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!( + "failed to read CODEX_BASE_INSTRUCTIONS_FILE '{}': {e}", + path + )); + sections.push(contents); + } + Ok(_) => { + // Explicitly disabled (empty or "-"): skip base instructions + } + Err(_) => { + sections.push(BASE_INSTRUCTIONS.to_string()); + } + } if let Some(ref user) = self.user_instructions { - sections.push(user); + sections.push(user.clone()); } if model.starts_with("gpt-4.1") { - sections.push(APPLY_PATCH_TOOL_INSTRUCTIONS); + sections.push(APPLY_PATCH_TOOL_INSTRUCTIONS.to_string()); } Cow::Owned(sections.join("\n")) }