openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-11 20:36:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add untrusted external messages to the Python SDK (#44086)

Browse Source 
## Why

Applications need to deliver content from other agents, tools, or services with tool-level authority, without treating it as user input or granting authorization.

## What changed

- Export `ExternalMessage` for sync and async `run(...)` and `turn(...)`, accepting text or structured content with a tool name and optional namespace. Send it through `toolOutput` and require CLI 0.151.0 or newer.
- Support starting a turn or joining an active regular turn while preserving external content as function output in history. Keep external messages separate from user-input lists and `steer(...)`.
- Give turn handles independent subscriptions, replaying completed items and latest usage to joining handles. Release consumed transient events and clean up subscriptions on closure, failure, or cancellation.
- Document the authority boundary and add sync and async examples.

## Testing

Add coverage for wire representations, input validation, runtime compatibility, tool authority across resume, active-turn joins, and tool-output truncation. Add subscription tests for replay, concurrent consumers, early completion, cancellation, and cleanup.

GitOrigin-RevId: 6106327085fd9c4bd11b71e20b3d8e74738b8bb5
This commit is contained in:
Ahmed Ibrahim
2026-09-09 06:46:01 +00:00
committed by copyberry
parent 8afccec87a
commit 1a4096e273
19 changed files with 1031 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/python/src/openai_codex/__init__.py
View File

@@ -23,6 +23,7 @@ from .api import (
ChatgptLoginHandle,
Codex,
DeviceCodeLoginHandle,
ExternalMessage,
ImageInput,
Input,
InputItem,
@@ -72,6 +73,7 @@ __all__ = [
"Input",
"InputItem",
"RunInput",
"ExternalMessage",
"TextInput",
"ImageInput",
"LocalImageInput",

34
sdk/python/src/openai_codex/_inputs.py
View File

@@ -1,7 +1,9 @@
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from .generated.v2_all import FunctionCallOutputContentItem, TurnToolOutput
from .models import JsonObject
@@ -42,9 +44,27 @@ class MentionInput:
path: str
@dataclass(slots=True)
class ExternalMessage:
"""Untrusted content supplied by another agent, tool, or application.
Content has tool-level authority, below user and developer instructions. It
does not establish user authorization or approval. Pass this as the whole
input to ``thread.run()`` or ``thread.turn()`` to start a turn or join an
active regular turn. ``tool_name`` identifies the tool delivering it.
``content`` accepts text or Responses-compatible function-output content
items. Structured items can be dictionaries; no generated wrapper is needed.
"""
tool_name: str
content: str | Sequence[JsonObject | FunctionCallOutputContentItem]
namespace: str | None = None
InputItem = TextInput | ImageInput | LocalImageInput | SkillInput | MentionInput
Input = list[InputItem] | InputItem
RunInput = Input | str
RunInput = Input | str | ExternalMessage
def _to_wire_item(item: InputItem) -> JsonObject:
@@ -67,7 +87,17 @@ def _to_wire_input(input: Input) -> list[JsonObject]:
return [_to_wire_item(input)]
def _normalize_run_input(input: RunInput) -> Input:
def _normalize_run_input(input: Input | str) -> Input:
if isinstance(input, str):
return TextInput(input)
return input
def _to_wire_turn_input(input: RunInput) -> tuple[list[JsonObject], TurnToolOutput | None]:
if isinstance(input, ExternalMessage):
if not isinstance(input.tool_name, str) or not input.tool_name.strip():
raise ValueError("ExternalMessage.tool_name must be a nonempty string")
return [], TurnToolOutput.model_validate(
{"name": input.tool_name, "namespace": input.namespace, "output": input.content}
)
return _to_wire_input(_normalize_run_input(input)), None

244
sdk/python/src/openai_codex/_message_router.py
View File

@@ -2,35 +2,107 @@ from __future__ import annotations
import queue
import threading
import weakref
from collections import deque
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Iterator
from ._goal import _GoalOperationState
from .errors import CodexError, map_jsonrpc_error
from .errors import CodexError, TransportClosedError, map_jsonrpc_error
from .generated.notification_registry import notification_turn_id
from .generated.v2_all import AccountLoginCompletedNotification
from .generated.v2_all import (
AccountLoginCompletedNotification,
ItemCompletedNotification,
ThreadTokenUsageUpdatedNotification,
)
from .models import JsonValue, Notification, UnknownNotification
ResponseQueueItem = JsonValue | BaseException
NotificationQueueItem = Notification | BaseException
@dataclass
class _TurnState:
id: str
thread_id: str | None = None
events: dict[int, NotificationQueueItem] = field(default_factory=dict)
first_event: int = 0
next_event: int = 0
subscribers: dict[object, int] = field(default_factory=dict)
subscribed: bool = False
completed_items: dict[str, Notification] = field(default_factory=dict)
usage: Notification | None = None
terminal: NotificationQueueItem | None = None
unclaimed: int = 0
completed: bool = False
class _TurnSubscription:
"""One consumer's result snapshot and cursor over shared unread events."""
def __init__(self, router: MessageRouter, state: _TurnState) -> None:
self._router = router
self._state = state
self._cursor = state.first_event
self._token = object()
state.subscribers[self._token] = self._cursor
state.subscribed = True
self._replay: deque[NotificationQueueItem] = deque(state.completed_items.values())
if state.usage is not None:
self._replay.append(state.usage)
if state.terminal is not None:
self._replay.append(state.terminal)
self._closed = False
self._release = weakref.finalize(
self, router._release_turn, weakref.ref(router), state, self._token
)
def next(self) -> Notification:
with self._router._turn_condition:
while not self._replay and self._cursor == self._state.next_event and not self._closed:
self._router._turn_condition.wait()
if self._closed:
raise TransportClosedError("Turn subscription closed")
if self._replay:
item = self._replay.popleft()
else:
item = self._state.events[self._cursor]
self._cursor += 1
self._state.subscribers[self._token] = self._cursor
self._router._prune_turn_events(self._state)
if isinstance(item, BaseException):
raise item
return item
def close(self) -> None:
with self._router._turn_condition:
self._closed = True
self._replay.clear()
self._router._turn_condition.notify_all()
self._release()
class MessageRouter:
"""Route reader-thread messages to the SDK operation waiting for them.
The app-server stdio transport is a single ordered stream, so only the
reader thread should consume stdout. This router keeps the rest of the SDK
from competing for that stream by giving each in-flight JSON-RPC request
and active turn stream its own queue.
its own queue and each turn consumer its own event cursor.
"""
def __init__(self) -> None:
"""Create empty response, turn, and global notification queues."""
self._lock = threading.Lock()
# GC can release abandoned subscriptions during another routing operation.
self._lock = threading.RLock()
self._response_waiters: dict[str, queue.Queue[ResponseQueueItem]] = {}
self._login_notifications: dict[str, queue.Queue[NotificationQueueItem]] = {}
self._pending_login_notifications: dict[str, deque[Notification]] = {}
self._turn_notifications: dict[str, queue.Queue[NotificationQueueItem]] = {}
self._pending_turn_notifications: dict[str, deque[Notification]] = {}
self._turn_condition = threading.Condition(self._lock)
self._turn_states: dict[str, _TurnState] = {}
self._turn_notifications: dict[str, _TurnSubscription | None] = {}
self._pending_turn_requests: dict[str, int] = {}
self._goal_operations: dict[str, _GoalOperationState] = {}
self._global_notifications: queue.Queue[NotificationQueueItem] = queue.Queue()
@@ -86,37 +158,119 @@ class MessageRouter:
raise item
return item
def register_turn(self, turn_id: str) -> None:
"""Register a queue for a turn stream and replay early events."""
turn_queue: queue.Queue[NotificationQueueItem] = queue.Queue()
@contextmanager
def pending_turn(self, thread_id: str) -> Iterator[None]:
"""Retain early completion while a turn/start response is in flight."""
with self._lock:
if turn_id in self._turn_notifications:
return
# A turn can emit events immediately after turn/start, before the
# caller receives the TurnHandle and starts streaming.
pending = self._pending_turn_notifications.pop(turn_id, deque())
self._turn_notifications[turn_id] = turn_queue
for notification in pending:
turn_queue.put(notification)
self._pending_turn_requests[thread_id] = (
self._pending_turn_requests.get(thread_id, 0) + 1
)
try:
yield
finally:
with self._lock:
self._pending_turn_requests[thread_id] -= 1
if self._pending_turn_requests[thread_id] == 0:
del self._pending_turn_requests[thread_id]
for state in list(self._turn_states.values()):
if state.thread_id == thread_id:
self._prune_turn_events(state)
self._discard_finished_turn(state)
def prepare_turn(self, turn_id: str, thread_id: str) -> None:
"""Reserve the returned turn for a handle or a low-level consumer."""
with self._lock:
state = self._turn_states.setdefault(turn_id, _TurnState(turn_id, thread_id))
state.thread_id = thread_id
state.unclaimed += 1
self._turn_notifications.setdefault(turn_id, None)
def _subscribe_turn_locked(self, turn_id: str) -> _TurnSubscription:
state = self._turn_states.setdefault(turn_id, _TurnState(turn_id))
state.unclaimed = max(0, state.unclaimed - 1)
return _TurnSubscription(self, state)
def subscribe_turn(self, turn_id: str) -> _TurnSubscription:
"""Attach a consumer with completed items, latest usage, and unread events."""
with self._lock:
return self._subscribe_turn_locked(turn_id)
@staticmethod
def _release_turn(
router_ref: weakref.ReferenceType[MessageRouter], state: _TurnState, token: object
) -> None:
router = router_ref()
if router is not None:
with router._lock:
state.subscribers.pop(token, None)
router._prune_turn_events(state)
router._discard_finished_turn(state)
def _prune_turn_events(self, state: _TurnState) -> None:
if (
not state.subscribed
or state.unclaimed
or self._pending_turn_requests.get(state.thread_id, 0)
):
return
consumed = min(state.subscribers.values(), default=state.next_event)
while state.first_event < consumed:
event = state.events.pop(state.first_event)
state.first_event += 1
# Late joins need the completed result, not every consumed token delta
# or intermediate usage update. Keep one snapshot entry per item.
if isinstance(event, BaseException) or event.method == "turn/completed":
state.terminal = event
elif isinstance(event.payload, ItemCompletedNotification):
item = event.payload.item
state.completed_items[getattr(item, "root", item).id] = event
elif isinstance(event.payload, ThreadTokenUsageUpdatedNotification):
state.usage = event
def _discard_finished_turn(self, state: _TurnState) -> None:
if (
state.completed
and not state.subscribers
and not state.unclaimed
and not self._pending_turn_requests.get(state.thread_id, 0)
):
self._turn_states.pop(state.id, None)
if self._turn_notifications.get(state.id) is None:
self._turn_notifications.pop(state.id, None)
state.events.clear()
state.completed_items.clear()
state.usage = None
state.terminal = None
def register_turn(self, turn_id: str) -> None:
"""Register the default consumer used by the low-level client API."""
with self._lock:
if self._turn_notifications.get(turn_id) is None:
self._turn_notifications[turn_id] = self._subscribe_turn_locked(turn_id)
def unregister_turn(self, turn_id: str) -> None:
"""Stop routing future turn events to the stream queue."""
"""Close only the low-level consumer, leaving other handles subscribed."""
with self._lock:
self._turn_notifications.pop(turn_id, None)
if turn_id not in self._turn_notifications:
return
subscription = self._turn_notifications.pop(turn_id)
if subscription is None and (state := self._turn_states.get(turn_id)) is not None:
state.unclaimed = max(0, state.unclaimed - 1)
self._prune_turn_events(state)
self._discard_finished_turn(state)
if subscription is not None:
subscription.close()
def next_turn_notification(self, turn_id: str) -> Notification:
"""Block until the next notification for a registered turn."""
"""Block until the next event for the default low-level consumer."""
with self._lock:
turn_queue = self._turn_notifications.get(turn_id)
if turn_queue is None:
raise RuntimeError(f"turn {turn_id!r} is not registered for streaming")
item = turn_queue.get()
if isinstance(item, BaseException):
raise item
return item
if turn_id not in self._turn_notifications:
raise RuntimeError(f"turn {turn_id!r} is not registered for streaming")
subscription = self._turn_notifications[turn_id]
if subscription is None:
subscription = self._subscribe_turn_locked(turn_id)
self._turn_notifications[turn_id] = subscription
return subscription.next()
def register_goal(self, thread_id: str) -> _GoalOperationState:
"""Register one thread-scoped logical goal operation before it starts."""
@@ -204,15 +358,16 @@ class MessageRouter:
self._global_notifications.put(notification)
return
with self._lock:
turn_queue = self._turn_notifications.get(turn_id)
if turn_queue is None:
if notification.method == "turn/completed":
self._pending_turn_notifications.pop(turn_id, None)
return
self._pending_turn_notifications.setdefault(turn_id, deque()).append(notification)
return
turn_queue.put(notification)
with self._turn_condition:
state = self._turn_states.setdefault(turn_id, _TurnState(turn_id, thread_id))
state.thread_id = thread_id or state.thread_id
state.events[state.next_event] = notification
state.next_event += 1
self._prune_turn_events(state)
if notification.method == "turn/completed":
state.completed = True
self._discard_finished_turn(state)
self._turn_condition.notify_all()
def fail_all(self, exc: BaseException) -> None:
"""Wake every blocked waiter when the reader thread exits."""
@@ -223,8 +378,13 @@ class MessageRouter:
login_queues = list(self._login_notifications.values())
self._login_notifications.clear()
self._pending_login_notifications.clear()
turn_queues = list(self._turn_notifications.values())
self._pending_turn_notifications.clear()
for state in list(self._turn_states.values()):
state.events[state.next_event] = exc
state.next_event += 1
self._prune_turn_events(state)
state.completed = True
self._discard_finished_turn(state)
self._turn_condition.notify_all()
goal_operations = list(self._goal_operations.values())
self._goal_operations.clear()
# Put the same transport failure into every queue so no SDK call blocks
@@ -233,8 +393,6 @@ class MessageRouter:
waiter.put(exc)
for login_queue in login_queues:
login_queue.put(exc)
for turn_queue in turn_queues:
turn_queue.put(exc)
for goal_operation in goal_operations:
goal_operation.fail(exc)
self._global_notifications.put(exc)

51
sdk/python/src/openai_codex/api.py
View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import AsyncIterator, Iterator
from ._approval_mode import (
@@ -11,6 +11,7 @@ from ._approval_mode import (
)
from ._initialize_metadata import validate_initialize_metadata
from ._inputs import (
ExternalMessage as ExternalMessage,
ImageInput as ImageInput,
Input as Input,
InputItem as InputItem,
@@ -21,6 +22,7 @@ from ._inputs import (
TextInput as TextInput,
_normalize_run_input,
_to_wire_input,
_to_wire_turn_input,
)
from ._login import (
AsyncChatgptLoginHandle,
@@ -32,6 +34,7 @@ from ._login import (
start_chatgpt_login,
start_device_code_login,
)
from ._message_router import _TurnSubscription
from ._run import (
TurnResult,
_collect_async_turn_result,
@@ -584,7 +587,8 @@ class Thread:
) -> TurnResult:
"""Run a complete turn and collect its final result.
Accepts the same input and options as turn().
Accepts the same input and options as turn(), including ExternalMessage
for untrusted external content with tool-level authority.
"""
turn = self.turn(
input,
@@ -618,17 +622,20 @@ class Thread:
summary: ReasoningSummary | None = None,
turn_service_tier: str | None = None,
) -> TurnHandle:
"""Start a turn and return a handle for streaming or control.
"""Start a turn or join an active regular turn and return its handle.
ExternalMessage supplies untrusted content with tool-level authority;
it does not establish user authorization or approval.
turn_service_tier applies only to this new turn; service_tier updates
the thread default. source labels what initiated a new turn and grants
no authority. Both turn_service_tier and source are ignored when joining.
"""
wire_input = _to_wire_input(_normalize_run_input(input))
wire_input, tool_output = _to_wire_turn_input(input)
approval_policy, approvals_reviewer = _approval_mode_override_settings(approval_mode)
params = TurnStartParams(
thread_id=self.id,
input=wire_input,
tool_output=tool_output,
approval_policy=approval_policy,
approvals_reviewer=approvals_reviewer,
cwd=cwd,
@@ -684,7 +691,8 @@ class AsyncThread:
) -> TurnResult:
"""Run a complete turn and collect its final result.
Accepts the same input and options as turn().
Accepts the same input and options as turn(), including ExternalMessage
for untrusted external content with tool-level authority.
"""
turn = await self.turn(
input,
@@ -718,18 +726,21 @@ class AsyncThread:
summary: ReasoningSummary | None = None,
turn_service_tier: str | None = None,
) -> AsyncTurnHandle:
"""Start a turn and return a handle for streaming or control.
"""Start a turn or join an active regular turn and return its handle.
ExternalMessage supplies untrusted content with tool-level authority;
it does not establish user authorization or approval.
turn_service_tier applies only to this new turn; service_tier updates
the thread default. source labels what initiated a new turn and grants
no authority. Both turn_service_tier and source are ignored when joining.
"""
wire_input = _to_wire_input(_normalize_run_input(input))
wire_input, tool_output = _to_wire_turn_input(input)
await self._codex._ensure_initialized()
approval_policy, approvals_reviewer = _approval_mode_override_settings(approval_mode)
params = TurnStartParams(
thread_id=self.id,
input=wire_input,
tool_output=tool_output,
approval_policy=approval_policy,
approvals_reviewer=approvals_reviewer,
cwd=cwd,
@@ -769,9 +780,13 @@ class TurnHandle:
_client: CodexClient
thread_id: str
id: str
_subscription: _TurnSubscription = field(init=False, repr=False, compare=False)
def steer(self, input: RunInput) -> TurnSteerResponse:
"""Send additional input to this active turn."""
def __post_init__(self) -> None:
self._subscription = self._client._subscribe_turn_notifications(self.id)
def steer(self, input: Input | str) -> TurnSteerResponse:
"""Send additional user input to this active turn."""
return self._client.turn_steer(
self.thread_id,
self.id,
@@ -784,10 +799,9 @@ class TurnHandle:
def stream(self) -> Iterator[Notification]:
"""Yield only notifications routed to this turn handle."""
self._client.register_turn_notifications(self.id)
try:
while True:
event = self._client.next_turn_notification(self.id)
event = self._subscription.next()
yield event
if (
event.method == "turn/completed"
@@ -796,7 +810,7 @@ class TurnHandle:
):
break
finally:
self._client.unregister_turn_notifications(self.id)
self._subscription.close()
def run(self) -> TurnResult:
"""Consume the turn stream and return its completed result."""
@@ -814,9 +828,13 @@ class AsyncTurnHandle:
_codex: AsyncCodex
thread_id: str
id: str
_subscription: _TurnSubscription = field(init=False, repr=False, compare=False)
async def steer(self, input: RunInput) -> TurnSteerResponse:
"""Send additional input to this active turn."""
def __post_init__(self) -> None:
self._subscription = self._codex._client._subscribe_turn_notifications(self.id)
async def steer(self, input: Input | str) -> TurnSteerResponse:
"""Send additional user input to this active turn."""
await self._codex._ensure_initialized()
return await self._codex._client.turn_steer(
self.thread_id,
@@ -832,10 +850,9 @@ class AsyncTurnHandle:
async def stream(self) -> AsyncIterator[Notification]:
"""Yield only notifications routed to this async turn handle."""
await self._codex._ensure_initialized()
self._codex._client.register_turn_notifications(self.id)
try:
while True:
event = await self._codex._client.next_turn_notification(self.id)
event = await asyncio.to_thread(self._subscription.next)
yield event
if (
event.method == "turn/completed"
@@ -844,7 +861,7 @@ class AsyncTurnHandle:
):
break
finally:
self._codex._client.unregister_turn_notifications(self.id)
self._subscription.close()
async def run(self) -> TurnResult:
"""Consume the turn stream and return its completed result."""

29
sdk/python/src/openai_codex/async_client.py
View File

@@ -3,12 +3,14 @@ from __future__ import annotations
import asyncio
import threading
from collections.abc import Iterator
from concurrent.futures import Future
from concurrent.futures import Future, ThreadPoolExecutor
from contextvars import copy_context
from typing import AsyncIterator, Callable, ParamSpec, TypeVar
from pydantic import BaseModel
from ._goal import _GoalOperationState
from ._message_router import _TurnSubscription
from .client import CodexClient, CodexConfig
from .generated.v2_all import (
AccountLoginCompletedNotification,
@@ -48,6 +50,9 @@ ModelT = TypeVar("ModelT", bound=BaseModel)
ParamsT = ParamSpec("ParamsT")
ReturnT = TypeVar("ReturnT")
# Bound workers while allowing cancellation cleanup to outlive an asyncio waiter.
_TURN_START_EXECUTOR = ThreadPoolExecutor(thread_name_prefix="codex-turn-start")
class AsyncCodexClient:
"""Async wrapper around CodexClient using thread offloading."""
@@ -97,6 +102,9 @@ class AsyncCodexClient:
"""Initialize the Codex session."""
return await self._call_sync(self._sync.initialize)
def _subscribe_turn_notifications(self, turn_id: str) -> _TurnSubscription:
return self._sync._subscribe_turn_notifications(turn_id)
def register_turn_notifications(self, turn_id: str) -> None:
"""Register a turn notification queue on the wrapped sync client."""
self._sync.register_turn_notifications(turn_id)
@@ -287,8 +295,23 @@ class AsyncCodexClient:
input_items: list[JsonObject] | JsonObject | str,
params: V2TurnStartParams | JsonObject | None = None,
) -> TurnStartResponse:
"""Start a turn using the wrapped sync client."""
return await self._call_sync(self._sync.turn_start, thread_id, input_items, params)
"""Start a turn, releasing an unclaimed result if the caller is cancelled."""
operation = _TURN_START_EXECUTOR.submit(
copy_context().run, self._sync.turn_start, thread_id, input_items, params
)
try:
return await asyncio.wrap_future(operation)
except asyncio.CancelledError:
def discard_cancelled_result(completed: Future[TurnStartResponse]) -> None:
try:
started = completed.result()
except BaseException:
return
self._sync._subscribe_turn_notifications(started.turn.id).close()
operation.add_done_callback(discard_cancelled_result)
raise
async def turn_interrupt(self, thread_id: str, turn_id: str) -> TurnInterruptResponse:
"""Interrupt a turn using the wrapped sync client."""

14
sdk/python/src/openai_codex/client.py
View File

@@ -15,7 +15,7 @@ from pydantic import BaseModel
from ._goal import _GoalOperationState
from ._initialize_metadata import _split_user_agent
from ._message_router import MessageRouter
from ._message_router import MessageRouter, _TurnSubscription
from ._runtime_requirements import CheckoutCapabilities, require_runtime_version
from ._version import __version__ as SDK_VERSION
from .errors import CodexError, InvalidRequestError, TransportClosedError
@@ -332,7 +332,7 @@ class CodexClient:
response_model: type[ModelT],
) -> ModelT:
runtime_fields = {
"turn/start": ("turnTrigger", "serviceTierForTurn"),
"turn/start": ("toolOutput", "turnTrigger", "serviceTierForTurn"),
"thread/resume": ("excludeTurns",),
"thread/fork": ("excludeTurns",),
}
@@ -407,6 +407,9 @@ class CodexClient:
"""Return the next routed notification for the requested login id."""
return self._router.next_login_notification(login_id)
def _subscribe_turn_notifications(self, turn_id: str) -> _TurnSubscription:
return self._router.subscribe_turn(turn_id)
def register_turn_notifications(self, turn_id: str) -> None:
"""Start routing notifications for one turn into its dedicated queue."""
self._router.register_turn(turn_id)
@@ -662,9 +665,10 @@ class CodexClient:
"threadId": thread_id,
"input": self._normalize_input_items(input_items),
}
started = self.request("turn/start", payload, response_model=TurnStartResponse)
self.register_turn_notifications(started.turn.id)
return started
with self._router.pending_turn(thread_id):
started = self.request("turn/start", payload, response_model=TurnStartResponse)
self._router.prepare_turn(started.turn.id, thread_id)
return started
@contextmanager
def _thread_start_lock(self, thread_id: str) -> Iterator[None]: